diff --git "a/wandb/run-20240328_133004-d0uv7bu9/logs/debug-internal.log" "b/wandb/run-20240328_133004-d0uv7bu9/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240328_133004-d0uv7bu9/logs/debug-internal.log" @@ -0,0 +1,2002 @@ +2024-03-28 13:30:04,910 INFO StreamThr :118 [internal.py:wandb_internal():86] W&B internal server running at pid: 118, started at: 2024-03-28 13:30:04.909963 +2024-03-28 13:30:04,912 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status +2024-03-28 13:30:05,251 INFO WriterThread:118 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/run-d0uv7bu9.wandb +2024-03-28 13:30:05,251 DEBUG SenderThread:118 [sender.py:send():378] send: header +2024-03-28 13:30:05,257 DEBUG SenderThread:118 [sender.py:send():378] send: run +2024-03-28 13:30:05,423 INFO SenderThread:118 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files +2024-03-28 13:30:05,423 INFO SenderThread:118 [sender.py:_start_run_threads():1099] run started: d0uv7bu9 with start time 1711632604.911076 +2024-03-28 13:30:05,425 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: check_version +2024-03-28 13:30:05,425 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: check_version +2024-03-28 13:30:05,513 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: run_start +2024-03-28 13:30:05,526 DEBUG HandlerThread:118 [system_info.py:__init__():26] System info init +2024-03-28 13:30:05,526 DEBUG HandlerThread:118 [system_info.py:__init__():41] System info init done +2024-03-28 13:30:05,526 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 13:30:05,526 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 13:30:05,527 INFO HandlerThread:118 [system_monitor.py:probe():214] Collecting system info +2024-03-28 13:30:05,527 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 13:30:05,527 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 13:30:05,528 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 13:30:05,529 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 13:30:05,531 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 13:30:05,542 DEBUG HandlerThread:118 [system_info.py:probe():150] Probing system +2024-03-28 13:30:05,545 DEBUG HandlerThread:118 [gitlib.py:_init_repo():56] git repository is invalid +2024-03-28 13:30:05,545 DEBUG HandlerThread:118 [system_info.py:probe():198] Probing system done +2024-03-28 13:30:05,545 DEBUG HandlerThread:118 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-28T13:30:05.542748', 'startedAt': '2024-03-28T13:30:04.903309', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'e1d09984cc69', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.202, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.202, 'min': 0.0, 'max': 0.0}, {'current': 2000.202, 'min': 0.0, 'max': 0.0}, {'current': 2000.202, 'min': 0.0, 'max': 0.0}, {'current': 2000.202, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5533.654685974121}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-03-28 13:30:05,545 INFO HandlerThread:118 [system_monitor.py:probe():224] Finished collecting system info +2024-03-28 13:30:05,545 INFO HandlerThread:118 [system_monitor.py:probe():227] Publishing system info +2024-03-28 13:30:05,545 DEBUG HandlerThread:118 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-03-28 13:30:06,425 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/conda-environment.yaml +2024-03-28 13:31:01,440 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/conda-environment.yaml +2024-03-28 13:31:01,973 DEBUG HandlerThread:118 [system_info.py:_save_conda():219] Saving conda packages done +2024-03-28 13:31:01,973 INFO HandlerThread:118 [system_monitor.py:probe():229] Finished publishing system info +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: python_packages +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 13:31:01,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:01,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:01,987 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: python_packages +2024-03-28 13:31:01,990 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 13:31:01,990 DEBUG SenderThread:118 [sender.py:send():378] send: config +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:01,992 WARNING SenderThread:118 [sender.py:send_metric():1316] Seen metric with glob (shouldn't happen) +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 13:31:01,992 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: stop_status +2024-03-28 13:31:02,085 DEBUG SenderThread:118 [sender.py:send():378] send: files +2024-03-28 13:31:02,085 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-metadata.json with policy now +2024-03-28 13:31:02,303 INFO wandb-upload_0:118 [upload_job.py:push():131] Uploaded file /tmp/tmpe9_g8s3rwandb/cmzqhrrs-wandb-metadata.json +2024-03-28 13:31:02,440 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-metadata.json +2024-03-28 13:31:02,441 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/requirements.txt +2024-03-28 13:31:02,441 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:31:04,441 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:31:05,531 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 13:31:05,533 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:31:05,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:06,442 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/config.yaml +2024-03-28 13:31:06,524 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:10,697 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:11,525 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:15,698 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:16,526 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:20,699 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:21,527 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:25,700 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:26,528 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:26,943 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:31:26,945 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:31:26,947 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:31:27,450 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:31:30,948 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:31,529 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:31,807 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:31:31,809 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:31,809 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:31,809 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:31,810 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 13:31:31,810 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:31:31,810 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:31:31,810 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:31:32,452 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:31:34,453 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:31:35,534 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:31:36,530 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:36,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:37,455 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/config.yaml +2024-03-28 13:31:41,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:41,653 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:46,532 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:46,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:51,533 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:51,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:31:56,534 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:31:56,655 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:01,535 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:01,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:05,535 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:32:06,536 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:07,535 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:11,537 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:12,536 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:16,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:17,537 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:21,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:22,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:24,841 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:32:24,842 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:32:24,842 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:32:24,844 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:32:25,475 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:32:26,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:27,845 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:29,963 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:32:29,964 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:32:29,964 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:32:29,965 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:32:30,477 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:32:31,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:32,478 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:32:33,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:35,535 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:32:36,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:38,537 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:41,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:43,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:46,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:48,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:51,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:53,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:32:56,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:32:58,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:01,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:03,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:05,537 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:33:06,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:09,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:11,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:14,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:16,551 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:19,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:21,552 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:23,103 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:33:23,104 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:33:23,104 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:33:23,107 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:33:23,498 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:33:25,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:26,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:28,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:33:28,130 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:33:28,131 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:33:28,131 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:33:28,500 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:33:30,364 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:30,501 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:33:31,554 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:35,365 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:35,538 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:33:36,555 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:40,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:41,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:45,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:46,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:50,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:51,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:33:55,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:33:56,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:00,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:01,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:05,539 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:34:06,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:06,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:11,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:11,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:16,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:16,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:21,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:21,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:22,264 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:34:22,265 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:34:22,265 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:34:22,267 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:34:22,521 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:34:26,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:27,247 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:34:27,248 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:34:27,248 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:34:27,249 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:27,249 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:34:27,523 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:34:28,523 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:34:31,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:32,487 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:35,539 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:34:36,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:37,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:41,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:42,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:46,571 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:47,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:51,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:52,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:34:56,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:34:57,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:01,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:02,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:05,540 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:35:06,575 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:08,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:11,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:13,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:16,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:18,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:21,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:21,860 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:35:21,861 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:35:21,862 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:35:21,862 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:35:22,545 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:35:23,863 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:26,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:26,869 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:35:26,870 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:35:26,870 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:35:26,871 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:35:27,547 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:35:28,548 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:35:29,111 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:31,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:34,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:35,541 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:35:36,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:39,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:41,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:44,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:46,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:49,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:51,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:54,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:35:56,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:35:59,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:01,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:04,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:05,542 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:36:06,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:10,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:11,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:15,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:16,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:20,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:20,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:36:20,889 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:36:20,889 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:36:20,889 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:36:21,568 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:36:21,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:25,891 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:25,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:36:25,893 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:36:25,894 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:36:25,894 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:36:26,570 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:36:26,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:28,570 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:36:31,132 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:31,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:35,543 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:36:36,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:36,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:41,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:41,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:46,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:46,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:51,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:51,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:36:56,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:36:56,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:01,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:01,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:05,544 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:37:06,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:07,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:11,599 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:12,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:16,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:17,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:19,508 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:37:19,509 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:37:19,509 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:37:19,509 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:37:19,589 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:37:21,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:23,510 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:24,498 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:37:24,500 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:37:24,500 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:37:24,501 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:37:24,591 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:37:26,592 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:37:26,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:28,743 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:31,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:33,744 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:35,545 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:37:36,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:39,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:41,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:44,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:46,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:49,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:51,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:54,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:37:56,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:37:59,550 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:01,608 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:04,551 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:05,546 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:38:06,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:10,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:11,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:15,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:16,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:18,949 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:38:18,950 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:38:18,950 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:38:18,951 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:38:19,612 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:38:20,951 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:21,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:23,947 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:38:23,948 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:38:23,948 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:38:23,949 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:38:24,614 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:38:26,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:26,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:26,614 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:38:31,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:31,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:35,547 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:38:36,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:36,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:41,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:41,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:46,550 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:46,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:51,551 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:51,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:38:56,552 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:38:56,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:01,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:01,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:05,548 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:39:06,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:07,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:11,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:12,550 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:16,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:17,084 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:39:17,085 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:39:17,085 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:39:17,086 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:39:17,632 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:39:18,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:21,624 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:22,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:39:22,088 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:39:22,088 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:39:22,088 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:39:22,634 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:39:23,436 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:24,635 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:39:26,625 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:28,437 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:31,626 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:33,437 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:35,556 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:39:36,627 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:38,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:41,628 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:43,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:46,629 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:48,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:51,630 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:53,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:39:56,631 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:39:58,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:01,632 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:03,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:05,550 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:40:06,633 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:09,552 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:11,634 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:14,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:16,234 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:40:16,235 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:40:16,235 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:40:16,237 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:40:16,635 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:16,654 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:40:20,238 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:21,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:40:21,229 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:40:21,230 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:40:21,230 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:40:21,636 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:21,656 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:40:23,657 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:40:25,460 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:26,637 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:30,461 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:31,638 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:35,462 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:35,551 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:40:36,639 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:40,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:41,640 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:45,554 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:46,641 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:50,555 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:51,642 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:40:55,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:40:56,643 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:00,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:01,644 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:05,555 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:41:06,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:06,645 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:11,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:11,646 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:14,722 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:41:14,723 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:41:14,724 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:41:14,724 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:41:15,676 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:41:16,647 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:16,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:19,721 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:41:19,722 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:41:19,723 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:41:19,723 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:41:20,678 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:41:21,649 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:21,678 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:41:21,968 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:26,650 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:26,970 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:31,651 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:31,971 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:35,554 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:41:36,652 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:37,555 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:41,653 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:42,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:46,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:47,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:51,655 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:52,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:41:56,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:41:57,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:01,657 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:02,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:05,555 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:42:06,658 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:08,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:11,659 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:13,237 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:42:13,239 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:42:13,239 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:42:13,239 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:42:13,698 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:42:14,240 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:16,660 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:18,235 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:42:18,236 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:42:18,237 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:42:18,237 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:42:18,700 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:42:19,475 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:21,661 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:21,701 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:42:24,476 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:26,662 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:29,477 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:31,663 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:34,478 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:35,556 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:42:36,664 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:39,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:41,665 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:44,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:46,666 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:49,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:51,667 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:54,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:42:56,668 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:42:59,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:01,669 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:04,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:05,557 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:43:06,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:10,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:11,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:43:11,671 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:43:11,671 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:11,672 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:43:11,672 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:43:11,720 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:43:15,673 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:16,677 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:16,681 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:43:16,682 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:43:16,682 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:43:16,683 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:43:16,721 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:43:19,723 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:43:20,932 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:21,678 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:25,933 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:26,679 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:30,934 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:31,680 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:35,558 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:43:36,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:36,681 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:41,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:41,681 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:46,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:46,682 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:51,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:51,683 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:43:56,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:43:56,684 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:01,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:01,685 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:05,559 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:44:06,686 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:07,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:10,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:44:10,658 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:44:10,658 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:44:10,658 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:44:10,741 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:44:11,687 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:12,659 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:15,669 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:44:15,670 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:44:15,671 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:44:15,671 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:44:15,743 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:44:16,693 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:17,744 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:44:17,906 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:21,694 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:22,907 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:26,695 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:27,908 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:31,696 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:32,909 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:35,559 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:44:36,697 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:38,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:41,698 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:43,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:46,698 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:48,562 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:51,699 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:53,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:44:56,700 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:44:58,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:01,701 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:03,565 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:05,560 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:45:06,702 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:09,290 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:45:09,291 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:45:09,292 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:45:09,292 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:09,292 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:45:09,762 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:45:11,703 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:14,293 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:14,302 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:45:14,303 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:45:14,304 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:45:14,304 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:45:14,764 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:45:16,704 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:17,766 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:45:19,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:21,705 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:24,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:26,706 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:29,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:31,707 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:34,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:35,561 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:45:36,707 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:40,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:41,708 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:45,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:46,709 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:50,565 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:51,710 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:45:55,566 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:45:56,711 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:00,567 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:01,712 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:05,562 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:46:06,563 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:06,713 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:08,397 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:46:08,399 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:46:08,399 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:46:08,399 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:46:08,784 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:46:11,714 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:12,401 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:13,399 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:46:13,400 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:46:13,401 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:46:13,401 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:46:13,786 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:46:15,787 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:46:16,715 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:17,635 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:21,716 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:22,636 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:26,717 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:27,637 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:31,717 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:32,638 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:35,563 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:46:36,718 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:38,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:41,719 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:43,565 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:46,720 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:48,566 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:51,721 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:53,567 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:46:56,722 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:46:58,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:01,723 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:03,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:05,564 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:47:06,430 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:47:06,431 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:47:06,432 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:47:06,432 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:47:06,724 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:06,805 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:47:09,433 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:11,420 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:47:11,422 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:47:11,422 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:47:11,423 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:47:11,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:11,807 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:47:13,808 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:47:14,707 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:16,726 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:19,708 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:21,727 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:24,709 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:26,728 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:29,710 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:31,729 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:34,711 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:35,565 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:47:36,730 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:40,566 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:41,731 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:45,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:46,732 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:50,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:51,733 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:47:55,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:47:56,734 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:00,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:01,735 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:04,672 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:48:04,673 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:48:04,674 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:48:04,674 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:48:04,826 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:48:05,566 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:48:06,567 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:06,736 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:09,689 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:48:09,690 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:48:09,691 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:48:09,691 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:48:09,827 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:48:11,737 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:11,828 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:48:11,927 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:16,738 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:16,928 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:21,739 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:21,929 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:26,740 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:26,930 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:31,741 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:31,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:35,567 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:48:36,742 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:37,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:41,743 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:42,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:46,744 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:47,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:51,745 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:52,571 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:48:56,746 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:48:57,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:01,747 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:02,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:04,960 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:49:04,962 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:49:04,962 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:49:04,963 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:49:05,568 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:49:05,847 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:49:06,748 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:08,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:09,967 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:49:09,969 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:49:09,969 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:49:09,970 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:49:10,849 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:49:11,749 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:11,849 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:49:14,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:16,750 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:19,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:21,751 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:24,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:26,752 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:29,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:31,753 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:34,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:35,569 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:49:36,754 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:39,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:41,756 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:44,571 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:46,757 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:49,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:51,758 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:54,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:49:56,759 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:49:59,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:01,764 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:03,234 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:50:03,235 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:50:03,235 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:50:03,236 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:50:03,871 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:50:05,236 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:05,570 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:50:06,765 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:08,260 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:50:08,261 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:50:08,261 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:50:08,262 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:50:08,873 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:50:09,873 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:50:10,530 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:11,766 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:15,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:16,767 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:20,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:21,768 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:25,532 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:26,769 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:30,533 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:31,770 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:35,534 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:35,571 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:50:36,771 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:40,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:41,772 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:45,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:46,773 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:50,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:51,774 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:50:55,575 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:50:56,775 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:00,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:01,776 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:02,964 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:51:02,965 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:51:02,966 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:51:02,966 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:51:03,892 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:51:05,572 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:51:06,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:06,777 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:07,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:51:07,956 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:51:07,956 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:51:07,957 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:51:08,895 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:51:09,895 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:51:11,778 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:12,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:16,779 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:17,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:21,780 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:22,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:26,781 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:27,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:31,782 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:32,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:35,578 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:51:36,783 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:37,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:41,784 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:42,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:46,785 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:47,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:51,786 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:52,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:51:56,786 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:51:57,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:01,787 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:02,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:02,942 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:52:02,943 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:52:02,943 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:52:02,943 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:52:03,915 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:52:05,574 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:52:06,788 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:07,934 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:52:07,935 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:52:07,936 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:52:07,936 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:07,939 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:52:08,916 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:52:09,917 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:52:11,789 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:13,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:16,790 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:18,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:21,791 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:23,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:26,792 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:28,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:31,793 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:33,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:35,575 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:52:36,794 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:38,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:41,795 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:43,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:46,796 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:48,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:51,797 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:53,578 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:52:56,798 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:52:58,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:01,800 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:01,918 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:53:01,919 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:53:01,919 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:53:01,921 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:53:01,935 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:53:03,922 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:05,575 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:53:06,801 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:06,919 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:53:06,921 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:53:06,921 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:53:06,922 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:53:06,936 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:53:09,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:09,938 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:53:11,802 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:14,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:16,803 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:19,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:21,804 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:24,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:26,805 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:29,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:31,806 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:34,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:35,576 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:53:36,807 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:39,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:41,808 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:44,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:46,809 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:49,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:51,810 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:54,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:53:56,811 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:53:59,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:01,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:54:01,413 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:54:01,413 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:54:01,414 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:54:01,812 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:01,959 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:54:05,415 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:05,577 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:54:06,410 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:54:06,412 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:54:06,412 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:54:06,412 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:54:06,813 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:06,961 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:54:07,961 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:54:10,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:11,814 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:15,655 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:16,815 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:20,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:21,816 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:25,657 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:26,817 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:30,658 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:31,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:35,578 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:54:36,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:36,819 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:41,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:41,820 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:46,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:46,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:51,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:51,822 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:54:56,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:54:56,823 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:00,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:55:00,612 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:55:00,613 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:55:00,613 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:55:00,982 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:55:01,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:01,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:05,579 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:55:05,624 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:55:05,625 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:55:05,626 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:55:05,626 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:55:05,984 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:55:06,830 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:06,865 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:07,985 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:55:11,831 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:11,866 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:16,832 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:16,867 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:21,833 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:21,868 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:26,834 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:26,869 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:31,835 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:31,870 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:35,580 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:55:36,836 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:37,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:41,837 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:42,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:46,838 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:47,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:51,839 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:52,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:56,840 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:55:57,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:55:59,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:55:59,024 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:55:59,025 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:55:59,025 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:56:00,004 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:56:01,840 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:03,026 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:04,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:56:04,048 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:56:04,048 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:56:04,049 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:56:05,007 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:56:05,581 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:56:06,007 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:56:06,841 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:08,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:11,843 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:13,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:16,844 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:18,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:21,845 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:23,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:26,846 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:28,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:31,847 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:33,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:35,582 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:56:36,848 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:39,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:41,849 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:44,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:46,850 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:49,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:51,851 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:54,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:56:56,852 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:56:58,473 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:56:58,474 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:56:58,474 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:56:58,475 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:56:59,026 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:57:00,476 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:01,853 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:03,480 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:57:03,482 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:57:03,482 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:57:03,482 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:57:04,028 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:57:05,583 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:57:05,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:06,029 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:57:06,854 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:10,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:11,855 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:15,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:16,856 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:20,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:21,857 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:25,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:26,858 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:30,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:31,859 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:35,584 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:57:36,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:36,860 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:41,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:41,861 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:46,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:46,862 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:51,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:51,863 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:56,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:57:56,865 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:57:57,901 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:57:57,902 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:57:57,902 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:57:57,903 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:57:58,048 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:58:01,865 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:01,904 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:02,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:58:02,893 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:58:02,894 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:58:02,894 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:58:03,050 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:58:04,051 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:58:05,585 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:58:06,867 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:07,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:11,868 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:12,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:16,869 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:17,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:21,870 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:22,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:26,871 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:27,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:31,872 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:32,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:35,586 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:58:36,873 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:38,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:41,874 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:43,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:46,875 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:48,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:51,876 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:53,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:58:56,877 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:58:57,668 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:58:57,669 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:58:57,669 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:58:57,670 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:58:58,072 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:58:58,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:01,878 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:02,665 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:59:02,666 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:59:02,667 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:59:02,667 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:59:03,074 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 13:59:03,974 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:04,074 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 13:59:05,587 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:59:06,879 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:09,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:11,880 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:14,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:16,881 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:19,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:21,882 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:24,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:26,884 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:29,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:31,885 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:34,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:35,588 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 13:59:36,886 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:40,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:41,887 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:45,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:46,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:50,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:51,889 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:55,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 13:59:56,652 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 13:59:56,653 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 13:59:56,653 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 13:59:56,653 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 13:59:56,889 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 13:59:57,094 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:00:00,654 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:01,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:00:01,657 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:00:01,658 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:00:01,658 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:00:01,890 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:02,095 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:00:04,096 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:00:05,589 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:00:06,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:06,891 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:11,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:11,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:16,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:16,893 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:21,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:21,895 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:26,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:26,896 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:31,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:31,897 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:35,590 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:00:36,898 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:37,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:41,899 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:42,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:46,900 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:47,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:51,901 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:52,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:00:55,487 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:00:55,488 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:00:55,489 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:00:55,489 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:00:56,115 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:00:56,902 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:00:58,490 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:00,493 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:01:00,494 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:01:00,494 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:01:00,495 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:01:01,117 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:01:01,907 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:02,117 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:01:03,909 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:05,591 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:01:06,907 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:09,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:11,908 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:14,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:16,909 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:19,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:21,910 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:24,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:26,911 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:29,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:31,912 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:34,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:35,592 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:01:36,913 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:40,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:41,915 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:45,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:46,915 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:50,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:51,917 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:54,889 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:01:54,891 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:01:54,891 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:01:54,891 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:01:55,136 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:01:55,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:01:56,917 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:01:59,902 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:01:59,903 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:01:59,903 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:01:59,904 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:02:00,138 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:02:01,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:01,923 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:02,139 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:02:05,592 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:02:06,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:06,924 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:11,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:11,926 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:16,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:16,927 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:21,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:21,928 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:26,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:26,929 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:31,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:31,930 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:35,594 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:02:36,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:37,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:41,932 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:42,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:46,933 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:47,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:51,934 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:52,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:53,401 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:02:53,403 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:02:53,403 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:02:53,403 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:02:54,158 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:02:56,935 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:02:58,405 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:02:58,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:02:58,412 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:02:58,412 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:02:58,413 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:02:59,160 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:03:00,160 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:03:01,936 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:03,646 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:05,600 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:03:06,937 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:09,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:11,938 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:14,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:16,939 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:19,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:21,940 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:24,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:26,941 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:29,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:31,942 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:34,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:35,596 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:03:36,943 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:40,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:41,944 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:45,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:46,945 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:50,599 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:51,946 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:52,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:03:52,428 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:03:52,428 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:03:52,430 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:03:53,181 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:03:56,431 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:03:56,947 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:03:57,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:03:57,429 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:03:57,429 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:03:57,430 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:03:58,183 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:03:59,183 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:04:01,661 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:01,947 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:05,597 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:04:06,948 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:07,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:11,950 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:12,599 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:16,951 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:17,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:21,952 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:22,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:26,953 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:27,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:31,954 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:32,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:35,599 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:04:36,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:38,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:41,956 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:43,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:46,957 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:48,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:51,957 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:52,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:04:52,186 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:04:52,186 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:04:52,187 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:04:52,204 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:04:54,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:04:56,958 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:04:57,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:04:57,172 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:04:57,172 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:04:57,172 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:04:57,206 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:04:59,206 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:04:59,410 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:01,959 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:04,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:05,598 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:05:06,960 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:09,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:11,961 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:14,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:16,962 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:19,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:21,963 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:24,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:26,964 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:29,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:31,965 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:34,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:35,599 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:05:36,966 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:40,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:41,967 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:45,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:46,968 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:50,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:51,836 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:05:51,837 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:05:51,837 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:05:51,838 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:05:51,969 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:52,226 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:05:55,839 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:05:56,847 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:05:56,849 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:05:56,849 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:05:56,850 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:05:56,970 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:05:57,228 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:05:59,229 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:06:01,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:01,971 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:05,600 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:06:06,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:06,972 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:11,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:11,973 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:16,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:16,974 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:21,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:21,975 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:26,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:26,976 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:31,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:31,977 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:35,602 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:06:36,978 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:37,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:41,979 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:42,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:46,980 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:47,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:50,684 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:06:50,685 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:06:50,686 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:06:50,686 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:06:51,249 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:06:51,981 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:52,687 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:06:55,685 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:06:55,686 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:06:55,687 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:06:55,687 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:06:56,251 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:06:56,982 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:06:57,251 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:06:57,929 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:01,983 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:02,930 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:05,603 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:07:06,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:08,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:11,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:13,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:16,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:18,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:21,987 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:23,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:26,988 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:28,608 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:31,989 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:33,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:35,604 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:07:36,990 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:39,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:41,991 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:44,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:46,992 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:49,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:49,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:07:49,986 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:07:49,986 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:07:49,987 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:07:50,271 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:07:51,994 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:54,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:07:54,986 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:07:54,986 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:07:54,987 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:07:54,987 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:07:55,273 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:07:57,000 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:07:57,274 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:08:00,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:02,002 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:05,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:05,604 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:08:07,003 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:10,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:12,004 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:15,606 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:17,005 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:20,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:22,005 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:25,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:27,006 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:30,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:32,007 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:35,606 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:08:36,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:37,008 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:41,608 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:42,009 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:46,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:47,010 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:48,772 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:08:48,773 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:08:48,773 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:08:48,774 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:08:49,293 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:08:51,775 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:08:52,011 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:53,769 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:08:53,770 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:08:53,771 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:08:53,771 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:08:54,295 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:08:55,295 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:08:57,012 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:08:57,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:02,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:02,024 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:05,607 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:09:07,014 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:07,608 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:12,015 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:12,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:17,016 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:17,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:22,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:22,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:27,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:27,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:32,018 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:32,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:35,608 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:09:37,019 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:38,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:42,020 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:43,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:47,021 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:47,317 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:09:47,318 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:09:47,318 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:09:47,319 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:09:48,316 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:09:49,319 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:52,022 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:52,322 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:09:52,323 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:09:52,323 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:09:52,324 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:09:53,317 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:09:54,998 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:09:55,318 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:09:57,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:09:59,999 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:02,025 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:05,000 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:05,609 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:10:07,026 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:10,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:12,027 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:15,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:17,028 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:20,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:22,029 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:25,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:27,030 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:30,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:32,030 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:35,610 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:10:36,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:37,032 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:41,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:42,033 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:46,056 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:10:46,057 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:10:46,057 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:10:46,058 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:10:46,337 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:10:47,034 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:47,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:51,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:10:51,083 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:10:51,083 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:10:51,084 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:10:51,339 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:10:52,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:52,317 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:10:53,340 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:10:57,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:10:57,318 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:02,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:02,319 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:05,611 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:11:07,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:07,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:12,039 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:12,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:17,040 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:17,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:22,041 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:22,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:27,042 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:27,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:32,043 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:32,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:35,611 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:11:37,044 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:38,612 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:42,045 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:43,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:45,626 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:11:45,627 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:11:45,627 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:11:45,628 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:11:46,359 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:11:47,046 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:48,629 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:50,609 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:11:50,610 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:11:50,611 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:11:50,611 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:11:51,361 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:11:52,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:53,362 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:11:53,844 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:11:57,048 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:11:58,845 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:02,049 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:03,845 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:05,612 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:12:07,050 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:09,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:12,051 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:14,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:17,052 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:19,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:22,053 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:24,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:27,054 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:29,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:32,055 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:34,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:35,613 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:12:37,056 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:40,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:42,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:44,949 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:12:44,950 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:12:44,951 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:12:44,951 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:12:45,382 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:12:45,952 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:47,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:49,958 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:12:49,959 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:12:49,959 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:12:49,959 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:12:50,384 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:12:51,288 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:51,384 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:12:52,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:12:56,289 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:12:57,060 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:01,290 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:02,061 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:05,614 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:13:06,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:07,062 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:11,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:12,063 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:16,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:17,064 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:21,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:22,065 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:26,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:27,066 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:31,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:32,067 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:35,615 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:13:37,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:37,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:42,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:42,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:43,952 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:13:43,954 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:13:43,954 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:13:43,955 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:13:44,405 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:13:47,070 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:47,956 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:48,966 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:13:48,967 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:13:48,967 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:13:48,968 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:13:49,407 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:13:51,408 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:13:52,071 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:53,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:13:57,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:13:58,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:02,073 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:03,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:05,616 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:14:07,074 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:08,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:12,075 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:13,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:17,076 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:18,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:22,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:23,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:27,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:28,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:32,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:33,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:35,617 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:14:37,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:39,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:42,082 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:42,450 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:14:42,451 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:14:42,451 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:14:42,452 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:14:43,429 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:14:45,453 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:47,082 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:47,455 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:14:47,456 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:14:47,456 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:14:47,456 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:14:48,431 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:14:49,431 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:14:50,694 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:52,084 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:14:55,695 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:14:57,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:00,696 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:02,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:05,618 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:15:06,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:07,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:11,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:12,088 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:16,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:17,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:21,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:22,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:26,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:27,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:31,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:32,092 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:35,618 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:15:37,093 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:37,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:41,075 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:15:41,076 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:15:41,077 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:15:41,077 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:15:41,450 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:15:42,093 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:43,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:46,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:15:46,069 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:15:46,070 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:15:46,070 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:15:46,451 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:15:47,094 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:47,452 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:15:48,455 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:52,095 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:53,456 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:15:57,096 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:15:58,456 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:02,097 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:03,457 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:05,619 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:16:07,098 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:08,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:12,099 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:13,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:17,100 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:18,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:22,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:23,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:27,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:28,624 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:32,106 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:33,625 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:35,620 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:16:37,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:39,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:40,290 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:16:40,291 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:16:40,292 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:16:40,292 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:16:40,471 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:16:42,108 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:45,293 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:45,297 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:16:45,298 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:16:45,298 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:16:45,299 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:16:45,473 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:16:47,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:47,474 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:16:50,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:52,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:16:55,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:16:57,110 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:00,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:02,111 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:05,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:05,621 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:17:07,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:10,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:12,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:15,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:16,746 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:17:16,748 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:17:16,748 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:17:16,748 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:17:17,114 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:17,486 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:17:20,750 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:21,775 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:17:21,776 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:17:21,777 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:17:21,777 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:17:22,042 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 14:17:22,044 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 14:17:22,045 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 14:17:22,046 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 14:17:22,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:17:22,047 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:17:22,047 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:17:22,048 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:17:22,048 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:17:22,048 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:17:22,049 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:17:22,059 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:17:22,059 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:17:22,059 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:17:22,060 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:17:22,060 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:17:22,060 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:17:22,060 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:17:22,061 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:17:22,061 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:17:22,062 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:17:22,062 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:17:22,063 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:17:22,064 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:17:22,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:22,242 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:17:22,242 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:17:22,242 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:17:22,242 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:17:22,243 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 14:17:22,243 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:17:22,243 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:17:22,244 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:17:22,254 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:17:22,254 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:17:22,254 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:17:22,254 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:17:22,255 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:17:22,255 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:17:22,255 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:17:22,255 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:17:22,256 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:17:22,257 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:17:22,259 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:17:22,259 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:17:22,260 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:17:22,488 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/wandb-summary.json +2024-03-28 14:17:23,797 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:17:23,797 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:17:23,797 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:17:23,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 14:17:23,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:17:23,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:17:23,798 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:17:23,799 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:17:23,809 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:17:23,809 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:17:23,809 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:17:23,809 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:17:25,810 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:27,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:30,811 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:32,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:35,812 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:37,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:40,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:42,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:42,496 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/config.yaml +2024-03-28 14:17:47,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:47,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:52,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:52,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:17:57,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:17:57,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:02,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:02,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:07,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:07,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:12,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:12,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:17,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:17,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:22,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:22,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:27,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:27,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:32,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:32,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:37,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:37,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:42,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:42,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:47,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:47,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:52,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:52,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:18:57,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:18:57,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:02,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:02,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:07,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:07,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:12,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:12,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:17,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:17,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:22,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:22,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:23,884 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:19:23,884 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:19:23,884 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:19:23,884 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:19:23,885 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:19:23,886 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:19:23,886 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:19:23,887 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:19:23,890 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:19:25,536 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:27,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:27,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:27,537 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:32,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:32,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:37,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:37,385 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:37,541 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:39,541 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:41,542 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_133004-d0uv7bu9/files/output.log +2024-03-28 14:19:42,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:42,404 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:47,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:47,424 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:48,751 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:19:48,751 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:19:48,751 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:19:48,751 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 14:19:48,752 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:19:48,752 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:19:48,753 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:19:48,754 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:19:48,766 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:19:48,767 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:19:48,767 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:19:48,767 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:19:48,767 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:19:48,767 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:19:48,767 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:19:48,768 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:19:48,769 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:19:48,769 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:19:48,770 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:19:48,770 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:19:48,771 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:19:48,796 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 14:19:48,796 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 14:19:48,796 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 14:19:48,796 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 14:19:48,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 14:19:48,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 14:19:48,797 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 14:19:48,798 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 14:19:48,811 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 14:19:48,811 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 14:19:48,811 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 14:19:48,812 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 14:19:52,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:52,813 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:19:57,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:19:57,814 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:02,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:02,815 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:07,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:07,816 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:12,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:12,817 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:17,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:17,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:22,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:22,819 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:27,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:27,820 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:32,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:32,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:37,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:37,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:42,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:42,822 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:47,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:47,823 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:52,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:52,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:20:57,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:20:57,686 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 14:20:57,686 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 14:20:57,687 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 14:20:57,687 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 14:20:57,688 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 14:20:57,688 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 14:20:57,689 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 14:20:57,690 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 14:20:57,691 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 14:20:57,825 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 14:21:02,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 14:21:02,826 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report