Training in progress, epoch 0

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": {
+    "base_model_class": "ViTForImageClassification",
+    "parent_library": "transformers.models.vit.modeling_vit"
+  },
+  "base_model_name_or_path": "/home/cc/polymorph/fine-tuning/results/train-lora/5/base_model",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier"
+  ],
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "value",
+    "query"
+  ],
+  "task_type": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:491fba07b6662ee825210fa5a398275c9b87e79fd07b27fb4591b8238de6b60c
+size 2372592

preprocessor_config.json

@@ -0,0 +1,22 @@
+{
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "ViTImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 224,
+    "width": 224
+  }
+}

runs/Jul17_16-22-01_gpu/events.out.tfevents.1721233323.gpu.2115683.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:303ef896a1277085a548aaa1b48e19fa060b55b06dfa7328fa16d0bf4b3e9344
+size 5374

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4d5975e4236cbb57f2744c0bb04888a6ce06aa5c95d1f870e20e7dec7483ba8
+size 5240

wandb/debug-internal.log

@@ -0,0 +1,76 @@
+2024-07-17 16:22:04,040 INFO    StreamThr :2116314 [internal.py:wandb_internal():85] W&B internal server running at pid: 2116314, started at: 2024-07-17 16:22:04.039368
+2024-07-17 16:22:04,043 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: status
+2024-07-17 16:22:04,046 INFO    WriterThread:2116314 [datastore.py:open_for_write():87] open: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/run-wglu07sk.wandb
+2024-07-17 16:22:04,050 DEBUG   SenderThread:2116314 [sender.py:send():379] send: header
+2024-07-17 16:22:04,051 DEBUG   SenderThread:2116314 [sender.py:send():379] send: run
+2024-07-17 16:22:04,205 INFO    SenderThread:2116314 [dir_watcher.py:__init__():211] watching files in: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files
+2024-07-17 16:22:04,205 INFO    SenderThread:2116314 [sender.py:_start_run_threads():1188] run started: wglu07sk with start time 1721233324.03891
+2024-07-17 16:22:04,218 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: check_version
+2024-07-17 16:22:04,219 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: check_version
+2024-07-17 16:22:04,281 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: run_start
+2024-07-17 16:22:04,332 DEBUG   HandlerThread:2116314 [system_info.py:__init__():26] System info init
+2024-07-17 16:22:04,332 DEBUG   HandlerThread:2116314 [system_info.py:__init__():41] System info init done
+2024-07-17 16:22:04,332 INFO    HandlerThread:2116314 [system_monitor.py:start():194] Starting system monitor
+2024-07-17 16:22:04,332 INFO    SystemMonitor:2116314 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-07-17 16:22:04,332 INFO    HandlerThread:2116314 [system_monitor.py:probe():214] Collecting system info
+2024-07-17 16:22:04,333 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started cpu monitoring
+2024-07-17 16:22:04,335 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started disk monitoring
+2024-07-17 16:22:04,336 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started gpu monitoring
+2024-07-17 16:22:04,337 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started memory monitoring
+2024-07-17 16:22:04,338 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started network monitoring
+2024-07-17 16:22:04,400 DEBUG   HandlerThread:2116314 [system_info.py:probe():152] Probing system
+2024-07-17 16:22:04,405 DEBUG   HandlerThread:2116314 [system_info.py:_probe_git():137] Probing git
+2024-07-17 16:22:04,416 DEBUG   HandlerThread:2116314 [system_info.py:_probe_git():145] Probing git done
+2024-07-17 16:22:04,416 DEBUG   HandlerThread:2116314 [system_info.py:probe():200] Probing system done
+2024-07-17 16:22:04,416 DEBUG   HandlerThread:2116314 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-101-generic-x86_64-with-glibc2.35', 'python': '3.11.9', 'heartbeatAt': '2024-07-17T16:22:04.400954', 'startedAt': '2024-07-17T16:22:04.032523', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/cc/polymorph/fine-tuning/main-lora-train.py', 'codePathLocal': 'main-lora-train.py', 'codePath': 'fine-tuning/main-lora-train.py', 'git': {'remote': 'https://github.com/inference-serving/polymorph.git', 'commit': 'e84189a37f0838a7e4ac1496b2345fe84c6a7683'}, 'email': '[email protected]', 'root': '/home/cc/polymorph', 'host': 'gpu', 'username': 'cc', 'executable': '/home/cc/miniconda3/envs/vision/bin/python', 'cpu_count': 24, 'cpu_count_logical': 48, 'cpu_freq': {'current': 2576.3446041666666, 'min': 1000.0, 'max': 3700.0}, 'cpu_freq_per_core': [{'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 1401.746, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}], 'disk': {'/': {'total': 208.95753479003906, 'used': 157.59302139282227}}, 'gpu': 'Quadro RTX 6000', 'gpu_count': 1, 'gpu_devices': [{'name': 'Quadro RTX 6000', 'memory_total': 25769803776}], 'memory': {'total': 187.4629783630371}}
+2024-07-17 16:22:04,417 INFO    HandlerThread:2116314 [system_monitor.py:probe():224] Finished collecting system info
+2024-07-17 16:22:04,417 INFO    HandlerThread:2116314 [system_monitor.py:probe():227] Publishing system info
+2024-07-17 16:22:04,417 DEBUG   HandlerThread:2116314 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2024-07-17 16:22:05,208 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/conda-environment.yaml
+2024-07-17 16:22:07,942 DEBUG   HandlerThread:2116314 [system_info.py:_save_conda():224] Saving conda packages done
+2024-07-17 16:22:07,943 INFO    HandlerThread:2116314 [system_monitor.py:probe():229] Finished publishing system info
+2024-07-17 16:22:07,953 DEBUG   SenderThread:2116314 [sender.py:send():379] send: files
+2024-07-17 16:22:07,953 INFO    SenderThread:2116314 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-07-17 16:22:08,106 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: python_packages
+2024-07-17 16:22:08,106 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: python_packages
+2024-07-17 16:22:08,107 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: stop_status
+2024-07-17 16:22:08,108 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: stop_status
+2024-07-17 16:22:08,112 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:08,159 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,159 DEBUG   SenderThread:2116314 [sender.py:send():379] send: config
+2024-07-17 16:22:08,161 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,162 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:08,162 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,162 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:08,162 WARNING SenderThread:2116314 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen)
+2024-07-17 16:22:08,162 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,163 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,163 DEBUG   SenderThread:2116314 [sender.py:send():379] send: config
+2024-07-17 16:22:08,206 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/conda-environment.yaml
+2024-07-17 16:22:08,207 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/requirements.txt
+2024-07-17 16:22:08,207 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/wandb-metadata.json
+2024-07-17 16:22:08,216 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: log_artifact
+2024-07-17 16:22:08,216 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: log_artifact
+2024-07-17 16:22:08,259 INFO    wandb-upload_0:2116314 [upload_job.py:push():130] Uploaded file /tmp/tmp6xd4bkanwandb/g8dj0aj4-wandb-metadata.json
+2024-07-17 16:22:08,735 INFO    wandb-upload_0:2116314 [upload_job.py:push():88] Uploaded file /tmp/tmpyvr0ja9x/model_architecture.txt
+2024-07-17 16:22:09,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:09,159 INFO    SenderThread:2116314 [sender.py:send_request_log_artifact():1518] logged artifact model-wglu07sk - {'id': 'QXJ0aWZhY3Q6OTkxMzgwOTg0', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjI4MjA3NTAwNA==', 'latestArtifact': None}}
+2024-07-17 16:22:09,160 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: status_report
+2024-07-17 16:22:09,207 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/output.log
+2024-07-17 16:22:10,107 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:11,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:11,208 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/output.log
+2024-07-17 16:22:12,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:13,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:13,209 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/output.log
+2024-07-17 16:22:13,550 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: partial_history
+2024-07-17 16:22:13,554 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,556 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,556 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send():379] send: history
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: summary_record
+2024-07-17 16:22:13,558 INFO    SenderThread:2116314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-07-17 16:22:14,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages

wandb/debug.log

@@ -0,0 +1,30 @@
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Current SDK version is 0.17.4
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Configure stats pid to 2115683
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Loading settings from /home/cc/.config/wandb/settings
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Loading settings from /home/cc/polymorph/fine-tuning/wandb/settings
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19'}
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'fine-tuning/main-lora-train.py', 'program_abspath': '/home/cc/polymorph/fine-tuning/main-lora-train.py', 'program': '/home/cc/polymorph/fine-tuning/main-lora-train.py'}
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:_log_setup():529] Logging user logs to /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/logs/debug.log
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:_log_setup():530] Logging internal logs to /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/logs/debug-internal.log
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:init():569] calling init triggers
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {}
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:init():619] starting backend
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:init():623] setting up manager
+2024-07-17 16:22:04,037 INFO    MainThread:2115683 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-07-17 16:22:04,038 INFO    MainThread:2115683 [wandb_init.py:init():631] backend started and connected
+2024-07-17 16:22:04,041 INFO    MainThread:2115683 [wandb_init.py:init():720] updated telemetry
+2024-07-17 16:22:04,050 INFO    MainThread:2115683 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-07-17 16:22:04,217 INFO    MainThread:2115683 [wandb_run.py:_on_init():2402] communicating current version
+2024-07-17 16:22:04,270 INFO    MainThread:2115683 [wandb_run.py:_on_init():2411] got version response 
+2024-07-17 16:22:04,270 INFO    MainThread:2115683 [wandb_init.py:init():804] starting run threads in backend
+2024-07-17 16:22:08,107 INFO    MainThread:2115683 [wandb_run.py:_console_start():2380] atexit reg
+2024-07-17 16:22:08,107 INFO    MainThread:2115683 [wandb_run.py:_redirect():2235] redirect: wrap_raw
+2024-07-17 16:22:08,107 INFO    MainThread:2115683 [wandb_run.py:_redirect():2300] Wrapping output streams.
+2024-07-17 16:22:08,107 INFO    MainThread:2115683 [wandb_run.py:_redirect():2325] Redirects installed.
+2024-07-17 16:22:08,108 INFO    MainThread:2115683 [wandb_init.py:init():847] run started, returning control to user process
+2024-07-17 16:22:08,110 INFO    MainThread:2115683 [wandb_run.py:_config_callback():1382] config_cb None None {'peft_config': {'default': {'peft_type': <PeftType.LORA: 'LORA'>, 'auto_mapping': None, 'base_model_name_or_path': '/home/cc/polymorph/fine-tuning/results/train-lora/5/base_model', 'revision': None, 'task_type': None, 'inference_mode': False, 'r': 16, 'target_modules': {'value', 'query'}, 'lora_alpha': 16, 'lora_dropout': 0.1, 'fan_in_fan_out': False, 'bias': 'none', 'use_rslora': False, 'modules_to_save': ['classifier'], 'init_lora_weights': True, 'layers_to_transform': None, 'layers_pattern': None, 'rank_pattern': {}, 'alpha_pattern': {}, 'megatron_config': None, 'megatron_core': 'megatron.core', 'loftq_config': {}, 'use_dora': False, 'layer_replication': None, 'runtime_config': {'ephemeral_gpu_offload': False}}}, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['ViTForImageClassification'], 'finetuning_task': None, 'id2label': {'0': 'airplane', '1': 'bird'}, 'label2id': {'airplane': '0', 'bird': '1'}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '/home/cc/polymorph/fine-tuning/results/train-lora/5/base_model', 'transformers_version': '4.42.3', 'model_type': 'vit', 'hidden_size': 768, 'num_hidden_layers': 12, 'num_attention_heads': 12, 'intermediate_size': 3072, 'hidden_act': 'gelu', 'hidden_dropout_prob': 0.0, 'attention_probs_dropout_prob': 0.0, 'initializer_range': 0.02, 'layer_norm_eps': 1e-12, 'image_size': 224, 'patch_size': 16, 'num_channels': 3, 'qkv_bias': True, 'encoder_stride': 16, 'output_dir': '/home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 128, 'per_device_eval_batch_size': 128, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.005, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/runs/Jul17_16-22-01_gpu', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': ['labels'], 'load_best_model_at_end': True, 'metric_for_best_model': 'accuracy', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'epoch', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False}
+2024-07-17 16:22:08,113 INFO    MainThread:2115683 [wandb_config.py:__setitem__():151] config set model/num_parameters = 86391556 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7fc944562610>>
+2024-07-17 16:22:08,113 INFO    MainThread:2115683 [wandb_run.py:_config_callback():1382] config_cb model/num_parameters 86391556 None

wandb/run-20240717_162204-wglu07sk/files/conda-environment.yaml

@@ -0,0 +1,248 @@
+name: vision
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - bzip2=1.0.8=h5eee18b_6
+  - ca-certificates=2024.3.11=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libuuid=1.41.5=h5eee18b_0
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.14=h5eee18b_0
+  - pip=24.0=py311h06a4308_0
+  - python=3.11.9=h955ad1f_0
+  - readline=8.2=h5eee18b_0
+  - setuptools=69.5.1=py311h06a4308_0
+  - sqlite=3.45.3=h5eee18b_0
+  - tk=8.6.14=h39e8969_0
+  - wheel=0.43.0=py311h06a4308_0
+  - xz=5.4.6=h5eee18b_1
+  - zlib=1.2.13=h5eee18b_1
+  - pip:
+      - absl-py==2.1.0
+      - accelerate==0.32.1
+      - aiohttp==3.9.5
+      - aiosignal==1.3.1
+      - annotated-types==0.7.0
+      - anyio==4.4.0
+      - argon2-cffi==23.1.0
+      - argon2-cffi-bindings==21.2.0
+      - arrow==1.3.0
+      - asttokens==2.4.1
+      - async-lru==2.0.4
+      - attrs==23.2.0
+      - babel==2.15.0
+      - beautifulsoup4==4.12.3
+      - black==24.4.2
+      - bleach==6.1.0
+      - certifi==2024.6.2
+      - cffi==1.16.0
+      - charset-normalizer==3.3.2
+      - click==8.1.7
+      - comm==0.2.2
+      - contourpy==1.2.1
+      - cycler==0.12.1
+      - datasets==2.20.0
+      - debugpy==1.8.1
+      - decorator==5.1.1
+      - defusedxml==0.7.1
+      - dill==0.3.8
+      - dnspython==2.6.1
+      - docker-pycreds==0.4.0
+      - einops==0.8.0
+      - email-validator==2.2.0
+      - evaluate==0.4.2
+      - executing==2.0.1
+      - fastapi==0.111.1
+      - fastapi-cli==0.0.4
+      - fastjsonschema==2.20.0
+      - filelock==3.15.4
+      - fonttools==4.53.0
+      - fqdn==1.5.1
+      - frozenlist==1.4.1
+      - fsspec==2024.5.0
+      - fvcore==0.1.6
+      - gitdb==4.0.11
+      - gitpython==3.1.43
+      - grpcio==1.64.1
+      - grpcio-tools==1.64.1
+      - h11==0.14.0
+      - httpcore==1.0.5
+      - httptools==0.6.1
+      - httpx==0.27.0
+      - huggingface-hub==0.23.4
+      - idna==3.7
+      - imageio==2.34.2
+      - imageio-ffmpeg==0.5.1
+      - install==1.3.5
+      - iopath==0.1.10
+      - ipykernel==6.29.4
+      - ipython==8.25.0
+      - ipywidgets==8.1.3
+      - isoduration==20.11.0
+      - jedi==0.19.1
+      - jinja2==3.1.4
+      - joblib==1.4.2
+      - json5==0.9.25
+      - jsonpointer==3.0.0
+      - jsonschema==4.22.0
+      - jsonschema-specifications==2023.12.1
+      - jupyter==1.0.0
+      - jupyter-client==8.6.2
+      - jupyter-console==6.6.3
+      - jupyter-core==5.7.2
+      - jupyter-events==0.10.0
+      - jupyter-lsp==2.2.5
+      - jupyter-server==2.14.1
+      - jupyter-server-terminals==0.5.3
+      - jupyterlab==4.2.2
+      - jupyterlab-pygments==0.3.0
+      - jupyterlab-server==2.27.2
+      - jupyterlab-widgets==3.0.11
+      - kaggle==1.6.14
+      - kiwisolver==1.4.5
+      - lazy-loader==0.4
+      - markdown==3.6
+      - markdown-it-py==3.0.0
+      - markupsafe==2.1.5
+      - matplotlib==3.9.0
+      - matplotlib-inline==0.1.7
+      - mdurl==0.1.2
+      - mistune==3.0.2
+      - mpmath==1.3.0
+      - multidict==6.0.5
+      - multiprocess==0.70.16
+      - mypy-extensions==1.0.0
+      - nbclient==0.10.0
+      - nbconvert==7.16.4
+      - nbformat==5.10.4
+      - nest-asyncio==1.6.0
+      - networkx==3.3
+      - notebook==7.2.1
+      - notebook-shim==0.2.4
+      - numpy==1.26.4
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==8.9.2.26
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-nccl-cu12==2.20.5
+      - nvidia-nvjitlink-cu12==12.5.40
+      - nvidia-nvtx-cu12==12.1.105
+      - opencv-python==4.10.0.84
+      - opencv-python-headless==4.10.0.84
+      - overrides==7.7.0
+      - packaging==24.1
+      - pandas==2.2.2
+      - pandocfilters==1.5.1
+      - parso==0.8.4
+      - pathspec==0.12.1
+      - peft==0.11.2.dev0
+      - pexpect==4.9.0
+      - pillow==10.3.0
+      - platformdirs==4.2.2
+      - portalocker==2.10.0
+      - prometheus-client==0.20.0
+      - prompt-toolkit==3.0.47
+      - protobuf==5.27.2
+      - psutil==6.0.0
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - py-cpuinfo==9.0.0
+      - pyarrow==16.1.0
+      - pyarrow-hotfix==0.6
+      - pycocotools==2.0.8
+      - pycparser==2.22
+      - pydantic==2.8.2
+      - pydantic-core==2.20.1
+      - pygments==2.18.0
+      - pyparsing==3.1.2
+      - python-dateutil==2.9.0.post0
+      - python-dotenv==1.0.1
+      - python-json-logger==2.0.7
+      - python-multipart==0.0.9
+      - python-slugify==8.0.4
+      - pytz==2024.1
+      - pyyaml==6.0.1
+      - pyzmq==26.0.3
+      - qtconsole==5.5.2
+      - qtpy==2.4.1
+      - ranger==0.10
+      - referencing==0.35.1
+      - regex==2024.5.15
+      - requests==2.32.3
+      - rfc3339-validator==0.1.4
+      - rfc3986-validator==0.1.1
+      - rich==13.7.1
+      - rpds-py==0.18.1
+      - safetensors==0.4.3
+      - scikit-image==0.24.0
+      - scikit-learn==1.5.0
+      - scipy==1.13.1
+      - seaborn==0.13.2
+      - send2trash==1.8.3
+      - sentry-sdk==2.7.1
+      - setproctitle==1.3.3
+      - shellingham==1.5.4
+      - simplejson==3.19.2
+      - six==1.16.0
+      - smmap==5.0.1
+      - sniffio==1.3.1
+      - soupsieve==2.5
+      - stack-data==0.6.3
+      - starlette==0.37.2
+      - sympy==1.12.1
+      - tabulate==0.9.0
+      - tensorboard==2.17.0
+      - tensorboard-data-server==0.7.2
+      - termcolor==2.4.0
+      - terminado==0.18.1
+      - text-unidecode==1.3
+      - thop==0.1.1-2209072238
+      - threadpoolctl==3.5.0
+      - tifffile==2024.6.18
+      - timm==1.0.7
+      - tinycss2==1.3.0
+      - tokenize-rt==5.2.0
+      - tokenizers==0.19.1
+      - torch==2.3.1
+      - torchinfo==1.6.3
+      - torchvision==0.18.1
+      - tornado==6.4.1
+      - tqdm==4.66.4
+      - traitlets==5.14.3
+      - transformers==4.42.3
+      - triton==2.3.1
+      - typer==0.12.3
+      - types-python-dateutil==2.9.0.20240316
+      - typing-extensions==4.12.2
+      - tzdata==2024.1
+      - ultralytics==8.2.41
+      - ultralytics-thop==2.0.0
+      - uri-template==1.3.0
+      - urllib3==2.2.2
+      - uvicorn==0.30.1
+      - uvloop==0.19.0
+      - vit-pytorch==1.7.0
+      - wandb==0.17.4
+      - watchfiles==0.22.0
+      - wcwidth==0.2.13
+      - webcolors==24.6.0
+      - webencodings==0.5.1
+      - websocket-client==1.8.0
+      - websockets==12.0
+      - werkzeug==3.0.3
+      - widgetsnbextension==4.0.11
+      - xxhash==3.4.1
+      - yacs==0.1.8
+      - yarl==1.9.4
+prefix: /home/cc/miniconda3/envs/vision

wandb/run-20240717_162204-wglu07sk/files/config.yaml

@@ -0,0 +1,35 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.11.9
+    cli_version: 0.17.4
+    framework: huggingface
+    huggingface_version: 4.42.3
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    start_time: 1721233324
+    t:
+      1:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      - 100
+      3:
+      - 13
+      - 23
+      4: 3.11.9
+      5: 0.17.4
+      6: 4.42.3
+      8:
+      - 2
+      - 5
+      13: linux-x86_64

wandb/run-20240717_162204-wglu07sk/files/output.log

@@ -0,0 +1,2 @@
+
+

wandb/run-20240717_162204-wglu07sk/files/requirements.txt

@@ -0,0 +1,225 @@
+Babel==2.15.0
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+PyYAML==6.0.1
+Pygments==2.18.0
+QtPy==2.4.1
+Send2Trash==1.8.3
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.32.1
+aiohttp==3.9.5
+aiosignal==1.3.1
+annotated-types==0.7.0
+anyio==4.4.0
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
+attrs==23.2.0
+beautifulsoup4==4.12.3
+black==24.4.2
+bleach==6.1.0
+certifi==2024.6.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+comm==0.2.2
+contourpy==1.2.1
+cycler==0.12.1
+datasets==2.20.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
+dnspython==2.6.1
+docker-pycreds==0.4.0
+einops==0.8.0
+email_validator==2.2.0
+evaluate==0.4.2
+executing==2.0.1
+fastapi-cli==0.0.4
+fastapi==0.111.1
+fastjsonschema==2.20.0
+filelock==3.15.4
+fonttools==4.53.0
+fqdn==1.5.1
+frozenlist==1.4.1
+fsspec==2024.5.0
+fvcore==0.1.6
+gitdb==4.0.11
+grpcio-tools==1.64.1
+grpcio==1.64.1
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.23.4
+idna==3.7
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+install==1.3.5
+iopath==0.1.10
+ipykernel==6.29.4
+ipython==8.25.0
+ipywidgets==8.1.3
+isoduration==20.11.0
+jedi==0.19.1
+joblib==1.4.2
+json5==0.9.25
+jsonpointer==3.0.0
+jsonschema-specifications==2023.12.1
+jsonschema==4.22.0
+jupyter-console==6.6.3
+jupyter-events==0.10.0
+jupyter-lsp==2.2.5
+jupyter==1.0.0
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+jupyter_server==2.14.1
+jupyter_server_terminals==0.5.3
+jupyterlab==4.2.2
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.2
+jupyterlab_widgets==3.0.11
+kaggle==1.6.14
+kiwisolver==1.4.5
+lazy_loader==0.4
+markdown-it-py==3.0.0
+matplotlib-inline==0.1.7
+matplotlib==3.9.0
+mdurl==0.1.2
+mistune==3.0.2
+mpmath==1.3.0
+multidict==6.0.5
+multiprocess==0.70.16
+mypy-extensions==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.3
+notebook==7.2.1
+notebook_shim==0.2.4
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.40
+nvidia-nvtx-cu12==12.1.105
+opencv-python-headless==4.10.0.84
+opencv-python==4.10.0.84
+overrides==7.7.0
+packaging==24.1
+pandas==2.2.2
+pandocfilters==1.5.1
+parso==0.8.4
+pathspec==0.12.1
+peft==0.11.2.dev0
+pexpect==4.9.0
+pillow==10.3.0
+pip==24.0
+platformdirs==4.2.2
+polymorph==0.1.0
+portalocker==2.10.0
+prometheus_client==0.20.0
+prompt_toolkit==3.0.47
+protobuf==5.27.2
+psutil==6.0.0
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow-hotfix==0.6
+pyarrow==16.1.0
+pycocotools==2.0.8
+pycparser==2.22
+pydantic==2.8.2
+pydantic_core==2.20.1
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-json-logger==2.0.7
+python-multipart==0.0.9
+python-slugify==8.0.4
+pytz==2024.1
+pyzmq==26.0.3
+qtconsole==5.5.2
+ranger==0.10
+referencing==0.35.1
+regex==2024.5.15
+requests==2.32.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rpds-py==0.18.1
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-learn==1.5.0
+scipy==1.13.1
+seaborn==0.13.2
+sentry-sdk==2.7.1
+setproctitle==1.3.3
+setuptools==69.5.1
+shellingham==1.5.4
+simplejson==3.19.2
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+stack-data==0.6.3
+starlette==0.37.2
+sympy==1.12.1
+tabulate==0.9.0
+tensorboard-data-server==0.7.2
+tensorboard==2.17.0
+termcolor==2.4.0
+terminado==0.18.1
+text-unidecode==1.3
+thop==0.1.1-2209072238
+threadpoolctl==3.5.0
+tifffile==2024.6.18
+timm==1.0.7
+tinycss2==1.3.0
+tokenize-rt==5.2.0
+tokenizers==0.19.1
+torch==2.3.1
+torchinfo==1.6.3
+torchvision==0.18.1
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.42.3
+triton==2.3.1
+typer==0.12.3
+types-python-dateutil==2.9.0.20240316
+typing_extensions==4.12.2
+tzdata==2024.1
+ultralytics-thop==2.0.0
+ultralytics==8.2.41
+uri-template==1.3.0
+urllib3==2.2.2
+uvicorn==0.30.1
+uvloop==0.19.0
+vit-pytorch==1.7.0
+wandb==0.17.4
+watchfiles==0.22.0
+wcwidth==0.2.13
+webcolors==24.6.0
+webencodings==0.5.1
+websocket-client==1.8.0
+websockets==12.0
+wheel==0.43.0
+widgetsnbextension==4.0.11
+xxhash==3.4.1
+yacs==0.1.8
+yarl==1.9.4

wandb/run-20240717_162204-wglu07sk/files/wandb-metadata.json

@@ -0,0 +1,288 @@
+{
+    "os": "Linux-5.15.0-101-generic-x86_64-with-glibc2.35",
+    "python": "3.11.9",
+    "heartbeatAt": "2024-07-17T16:22:04.400954",
+    "startedAt": "2024-07-17T16:22:04.032523",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "/home/cc/polymorph/fine-tuning/main-lora-train.py",
+    "codePathLocal": "main-lora-train.py",
+    "codePath": "fine-tuning/main-lora-train.py",
+    "git": {
+        "remote": "https://github.com/inference-serving/polymorph.git",
+        "commit": "e84189a37f0838a7e4ac1496b2345fe84c6a7683"
+    },
+    "email": "[email protected]",
+    "root": "/home/cc/polymorph",
+    "host": "gpu",
+    "username": "cc",
+    "executable": "/home/cc/miniconda3/envs/vision/bin/python",
+    "cpu_count": 24,
+    "cpu_count_logical": 48,
+    "cpu_freq": {
+        "current": 2576.3446041666666,
+        "min": 1000.0,
+        "max": 3700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 1401.746,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        },
+        {
+            "current": 2600.0,
+            "min": 1000.0,
+            "max": 3700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 208.95753479003906,
+            "used": 157.59302139282227
+        }
+    },
+    "gpu": "Quadro RTX 6000",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "Quadro RTX 6000",
+            "memory_total": 25769803776
+        }
+    ],
+    "memory": {
+        "total": 187.4629783630371
+    }
+}

wandb/run-20240717_162204-wglu07sk/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/loss": 0.5239526629447937, "eval/accuracy": 0.97, "eval/runtime": 0.8085, "eval/samples_per_second": 247.358, "eval/steps_per_second": 2.474, "train/epoch": 0.5714285714285714, "train/global_step": 1, "_timestamp": 1721233333.5496378, "_runtime": 9.510727882385254, "_step": 0}

wandb/run-20240717_162204-wglu07sk/logs/debug-internal.log

@@ -0,0 +1,76 @@
+2024-07-17 16:22:04,040 INFO    StreamThr :2116314 [internal.py:wandb_internal():85] W&B internal server running at pid: 2116314, started at: 2024-07-17 16:22:04.039368
+2024-07-17 16:22:04,043 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: status
+2024-07-17 16:22:04,046 INFO    WriterThread:2116314 [datastore.py:open_for_write():87] open: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/run-wglu07sk.wandb
+2024-07-17 16:22:04,050 DEBUG   SenderThread:2116314 [sender.py:send():379] send: header
+2024-07-17 16:22:04,051 DEBUG   SenderThread:2116314 [sender.py:send():379] send: run
+2024-07-17 16:22:04,205 INFO    SenderThread:2116314 [dir_watcher.py:__init__():211] watching files in: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files
+2024-07-17 16:22:04,205 INFO    SenderThread:2116314 [sender.py:_start_run_threads():1188] run started: wglu07sk with start time 1721233324.03891
+2024-07-17 16:22:04,218 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: check_version
+2024-07-17 16:22:04,219 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: check_version
+2024-07-17 16:22:04,281 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: run_start
+2024-07-17 16:22:04,332 DEBUG   HandlerThread:2116314 [system_info.py:__init__():26] System info init
+2024-07-17 16:22:04,332 DEBUG   HandlerThread:2116314 [system_info.py:__init__():41] System info init done
+2024-07-17 16:22:04,332 INFO    HandlerThread:2116314 [system_monitor.py:start():194] Starting system monitor
+2024-07-17 16:22:04,332 INFO    SystemMonitor:2116314 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-07-17 16:22:04,332 INFO    HandlerThread:2116314 [system_monitor.py:probe():214] Collecting system info
+2024-07-17 16:22:04,333 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started cpu monitoring
+2024-07-17 16:22:04,335 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started disk monitoring
+2024-07-17 16:22:04,336 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started gpu monitoring
+2024-07-17 16:22:04,337 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started memory monitoring
+2024-07-17 16:22:04,338 INFO    SystemMonitor:2116314 [interfaces.py:start():188] Started network monitoring
+2024-07-17 16:22:04,400 DEBUG   HandlerThread:2116314 [system_info.py:probe():152] Probing system
+2024-07-17 16:22:04,405 DEBUG   HandlerThread:2116314 [system_info.py:_probe_git():137] Probing git
+2024-07-17 16:22:04,416 DEBUG   HandlerThread:2116314 [system_info.py:_probe_git():145] Probing git done
+2024-07-17 16:22:04,416 DEBUG   HandlerThread:2116314 [system_info.py:probe():200] Probing system done
+2024-07-17 16:22:04,416 DEBUG   HandlerThread:2116314 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-101-generic-x86_64-with-glibc2.35', 'python': '3.11.9', 'heartbeatAt': '2024-07-17T16:22:04.400954', 'startedAt': '2024-07-17T16:22:04.032523', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/cc/polymorph/fine-tuning/main-lora-train.py', 'codePathLocal': 'main-lora-train.py', 'codePath': 'fine-tuning/main-lora-train.py', 'git': {'remote': 'https://github.com/inference-serving/polymorph.git', 'commit': 'e84189a37f0838a7e4ac1496b2345fe84c6a7683'}, 'email': '[email protected]', 'root': '/home/cc/polymorph', 'host': 'gpu', 'username': 'cc', 'executable': '/home/cc/miniconda3/envs/vision/bin/python', 'cpu_count': 24, 'cpu_count_logical': 48, 'cpu_freq': {'current': 2576.3446041666666, 'min': 1000.0, 'max': 3700.0}, 'cpu_freq_per_core': [{'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 1401.746, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}, {'current': 2600.0, 'min': 1000.0, 'max': 3700.0}], 'disk': {'/': {'total': 208.95753479003906, 'used': 157.59302139282227}}, 'gpu': 'Quadro RTX 6000', 'gpu_count': 1, 'gpu_devices': [{'name': 'Quadro RTX 6000', 'memory_total': 25769803776}], 'memory': {'total': 187.4629783630371}}
+2024-07-17 16:22:04,417 INFO    HandlerThread:2116314 [system_monitor.py:probe():224] Finished collecting system info
+2024-07-17 16:22:04,417 INFO    HandlerThread:2116314 [system_monitor.py:probe():227] Publishing system info
+2024-07-17 16:22:04,417 DEBUG   HandlerThread:2116314 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2024-07-17 16:22:05,208 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/conda-environment.yaml
+2024-07-17 16:22:07,942 DEBUG   HandlerThread:2116314 [system_info.py:_save_conda():224] Saving conda packages done
+2024-07-17 16:22:07,943 INFO    HandlerThread:2116314 [system_monitor.py:probe():229] Finished publishing system info
+2024-07-17 16:22:07,953 DEBUG   SenderThread:2116314 [sender.py:send():379] send: files
+2024-07-17 16:22:07,953 INFO    SenderThread:2116314 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-07-17 16:22:08,106 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: python_packages
+2024-07-17 16:22:08,106 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: python_packages
+2024-07-17 16:22:08,107 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: stop_status
+2024-07-17 16:22:08,108 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: stop_status
+2024-07-17 16:22:08,112 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:08,159 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,159 DEBUG   SenderThread:2116314 [sender.py:send():379] send: config
+2024-07-17 16:22:08,161 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,162 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:08,162 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,162 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:08,162 WARNING SenderThread:2116314 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen)
+2024-07-17 16:22:08,162 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,163 DEBUG   SenderThread:2116314 [sender.py:send():379] send: telemetry
+2024-07-17 16:22:08,163 DEBUG   SenderThread:2116314 [sender.py:send():379] send: config
+2024-07-17 16:22:08,206 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/conda-environment.yaml
+2024-07-17 16:22:08,207 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/requirements.txt
+2024-07-17 16:22:08,207 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/wandb-metadata.json
+2024-07-17 16:22:08,216 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: log_artifact
+2024-07-17 16:22:08,216 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: log_artifact
+2024-07-17 16:22:08,259 INFO    wandb-upload_0:2116314 [upload_job.py:push():130] Uploaded file /tmp/tmp6xd4bkanwandb/g8dj0aj4-wandb-metadata.json
+2024-07-17 16:22:08,735 INFO    wandb-upload_0:2116314 [upload_job.py:push():88] Uploaded file /tmp/tmpyvr0ja9x/model_architecture.txt
+2024-07-17 16:22:09,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:09,159 INFO    SenderThread:2116314 [sender.py:send_request_log_artifact():1518] logged artifact model-wglu07sk - {'id': 'QXJ0aWZhY3Q6OTkxMzgwOTg0', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjI4MjA3NTAwNA==', 'latestArtifact': None}}
+2024-07-17 16:22:09,160 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: status_report
+2024-07-17 16:22:09,207 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/output.log
+2024-07-17 16:22:10,107 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:11,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:11,208 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/output.log
+2024-07-17 16:22:12,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:13,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages
+2024-07-17 16:22:13,209 INFO    Thread-12 :2116314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/files/output.log
+2024-07-17 16:22:13,550 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: partial_history
+2024-07-17 16:22:13,554 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,556 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,556 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send():379] send: metric
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send():379] send: history
+2024-07-17 16:22:13,557 DEBUG   SenderThread:2116314 [sender.py:send_request():406] send_request: summary_record
+2024-07-17 16:22:13,558 INFO    SenderThread:2116314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-07-17 16:22:14,108 DEBUG   HandlerThread:2116314 [handler.py:handle_request():158] handle_request: internal_messages

wandb/run-20240717_162204-wglu07sk/logs/debug.log

@@ -0,0 +1,30 @@
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Current SDK version is 0.17.4
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Configure stats pid to 2115683
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Loading settings from /home/cc/.config/wandb/settings
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Loading settings from /home/cc/polymorph/fine-tuning/wandb/settings
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19'}
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'fine-tuning/main-lora-train.py', 'program_abspath': '/home/cc/polymorph/fine-tuning/main-lora-train.py', 'program': '/home/cc/polymorph/fine-tuning/main-lora-train.py'}
+2024-07-17 16:22:04,034 INFO    MainThread:2115683 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:_log_setup():529] Logging user logs to /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/logs/debug.log
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:_log_setup():530] Logging internal logs to /home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/wandb/run-20240717_162204-wglu07sk/logs/debug-internal.log
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:init():569] calling init triggers
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {}
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:init():619] starting backend
+2024-07-17 16:22:04,035 INFO    MainThread:2115683 [wandb_init.py:init():623] setting up manager
+2024-07-17 16:22:04,037 INFO    MainThread:2115683 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-07-17 16:22:04,038 INFO    MainThread:2115683 [wandb_init.py:init():631] backend started and connected
+2024-07-17 16:22:04,041 INFO    MainThread:2115683 [wandb_init.py:init():720] updated telemetry
+2024-07-17 16:22:04,050 INFO    MainThread:2115683 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-07-17 16:22:04,217 INFO    MainThread:2115683 [wandb_run.py:_on_init():2402] communicating current version
+2024-07-17 16:22:04,270 INFO    MainThread:2115683 [wandb_run.py:_on_init():2411] got version response 
+2024-07-17 16:22:04,270 INFO    MainThread:2115683 [wandb_init.py:init():804] starting run threads in backend
+2024-07-17 16:22:08,107 INFO    MainThread:2115683 [wandb_run.py:_console_start():2380] atexit reg
+2024-07-17 16:22:08,107 INFO    MainThread:2115683 [wandb_run.py:_redirect():2235] redirect: wrap_raw
+2024-07-17 16:22:08,107 INFO    MainThread:2115683 [wandb_run.py:_redirect():2300] Wrapping output streams.
+2024-07-17 16:22:08,107 INFO    MainThread:2115683 [wandb_run.py:_redirect():2325] Redirects installed.
+2024-07-17 16:22:08,108 INFO    MainThread:2115683 [wandb_init.py:init():847] run started, returning control to user process
+2024-07-17 16:22:08,110 INFO    MainThread:2115683 [wandb_run.py:_config_callback():1382] config_cb None None {'peft_config': {'default': {'peft_type': <PeftType.LORA: 'LORA'>, 'auto_mapping': None, 'base_model_name_or_path': '/home/cc/polymorph/fine-tuning/results/train-lora/5/base_model', 'revision': None, 'task_type': None, 'inference_mode': False, 'r': 16, 'target_modules': {'value', 'query'}, 'lora_alpha': 16, 'lora_dropout': 0.1, 'fan_in_fan_out': False, 'bias': 'none', 'use_rslora': False, 'modules_to_save': ['classifier'], 'init_lora_weights': True, 'layers_to_transform': None, 'layers_pattern': None, 'rank_pattern': {}, 'alpha_pattern': {}, 'megatron_config': None, 'megatron_core': 'megatron.core', 'loftq_config': {}, 'use_dora': False, 'layer_replication': None, 'runtime_config': {'ephemeral_gpu_offload': False}}}, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['ViTForImageClassification'], 'finetuning_task': None, 'id2label': {'0': 'airplane', '1': 'bird'}, 'label2id': {'airplane': '0', 'bird': '1'}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '/home/cc/polymorph/fine-tuning/results/train-lora/5/base_model', 'transformers_version': '4.42.3', 'model_type': 'vit', 'hidden_size': 768, 'num_hidden_layers': 12, 'num_attention_heads': 12, 'intermediate_size': 3072, 'hidden_act': 'gelu', 'hidden_dropout_prob': 0.0, 'attention_probs_dropout_prob': 0.0, 'initializer_range': 0.02, 'layer_norm_eps': 1e-12, 'image_size': 224, 'patch_size': 16, 'num_channels': 3, 'qkv_bias': True, 'encoder_stride': 16, 'output_dir': '/home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 128, 'per_device_eval_batch_size': 128, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.005, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19/runs/Jul17_16-22-01_gpu', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/home/cc/polymorph/fine-tuning/results/train-lora/5/loras/19', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': ['labels'], 'load_best_model_at_end': True, 'metric_for_best_model': 'accuracy', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': 'epoch', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False}
+2024-07-17 16:22:08,113 INFO    MainThread:2115683 [wandb_config.py:__setitem__():151] config set model/num_parameters = 86391556 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7fc944562610>>
+2024-07-17 16:22:08,113 INFO    MainThread:2115683 [wandb_run.py:_config_callback():1382] config_cb model/num_parameters 86391556 None