diff --git "a/wandb/run-20240328_025827-lnd0ipu3/logs/debug-internal.log" "b/wandb/run-20240328_025827-lnd0ipu3/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240328_025827-lnd0ipu3/logs/debug-internal.log" @@ -0,0 +1,2836 @@ +2024-03-28 02:58:27,478 INFO StreamThr :118 [internal.py:wandb_internal():86] W&B internal server running at pid: 118, started at: 2024-03-28 02:58:27.477747 +2024-03-28 02:58:27,480 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status +2024-03-28 02:58:27,828 INFO WriterThread:118 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/run-lnd0ipu3.wandb +2024-03-28 02:58:27,829 DEBUG SenderThread:118 [sender.py:send():378] send: header +2024-03-28 02:58:27,833 DEBUG SenderThread:118 [sender.py:send():378] send: run +2024-03-28 02:58:27,999 INFO SenderThread:118 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files +2024-03-28 02:58:28,000 INFO SenderThread:118 [sender.py:_start_run_threads():1099] run started: lnd0ipu3 with start time 1711594707.478978 +2024-03-28 02:58:28,001 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: check_version +2024-03-28 02:58:28,001 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: check_version +2024-03-28 02:58:28,091 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: run_start +2024-03-28 02:58:28,103 DEBUG HandlerThread:118 [system_info.py:__init__():26] System info init +2024-03-28 02:58:28,104 DEBUG HandlerThread:118 [system_info.py:__init__():41] System info init done +2024-03-28 02:58:28,104 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 02:58:28,104 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 02:58:28,104 INFO HandlerThread:118 [system_monitor.py:probe():214] Collecting system info +2024-03-28 02:58:28,105 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 02:58:28,105 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 02:58:28,106 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 02:58:28,107 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 02:58:28,108 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 02:58:28,118 DEBUG HandlerThread:118 [system_info.py:probe():150] Probing system +2024-03-28 02:58:28,121 DEBUG HandlerThread:118 [gitlib.py:_init_repo():56] git repository is invalid +2024-03-28 02:58:28,121 DEBUG HandlerThread:118 [system_info.py:probe():198] Probing system done +2024-03-28 02:58:28,121 DEBUG HandlerThread:118 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-28T02:58:28.119009', 'startedAt': '2024-03-28T02:58:27.470892', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'dbef23be6360', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.196, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.196, 'min': 0.0, 'max': 0.0}, {'current': 2000.196, 'min': 0.0, 'max': 0.0}, {'current': 2000.196, 'min': 0.0, 'max': 0.0}, {'current': 2000.196, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5533.654880523682}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-03-28 02:58:28,121 INFO HandlerThread:118 [system_monitor.py:probe():224] Finished collecting system info +2024-03-28 02:58:28,121 INFO HandlerThread:118 [system_monitor.py:probe():227] Publishing system info +2024-03-28 02:58:28,121 DEBUG HandlerThread:118 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-03-28 02:58:29,002 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/conda-environment.yaml +2024-03-28 02:59:25,020 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/conda-environment.yaml +2024-03-28 02:59:25,069 DEBUG HandlerThread:118 [system_info.py:_save_conda():219] Saving conda packages done +2024-03-28 02:59:25,069 INFO HandlerThread:118 [system_monitor.py:probe():229] Finished publishing system info +2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: python_packages +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:25,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:25,082 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: python_packages +2024-03-28 02:59:25,085 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 02:59:25,085 DEBUG SenderThread:118 [sender.py:send():378] send: config +2024-03-28 02:59:25,087 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 02:59:25,087 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 02:59:25,087 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: stop_status +2024-03-28 02:59:25,123 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 02:59:25,123 WARNING SenderThread:118 [sender.py:send_metric():1316] Seen metric with glob (shouldn't happen) +2024-03-28 02:59:25,123 DEBUG SenderThread:118 [sender.py:send():378] send: telemetry +2024-03-28 02:59:25,124 DEBUG SenderThread:118 [sender.py:send():378] send: files +2024-03-28 02:59:25,124 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-metadata.json with policy now +2024-03-28 02:59:25,326 INFO wandb-upload_0:118 [upload_job.py:push():131] Uploaded file /tmp/tmpcqrallmywandb/ogr9175t-wandb-metadata.json +2024-03-28 02:59:26,020 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/requirements.txt +2024-03-28 02:59:26,021 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 02:59:26,021 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-metadata.json +2024-03-28 02:59:28,021 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 02:59:28,108 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 02:59:28,110 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 02:59:28,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:29,021 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/config.yaml +2024-03-28 02:59:29,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:33,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:34,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:38,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:39,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:43,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:44,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:48,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:49,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:53,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:54,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 02:59:58,111 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 02:59:59,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:59:59,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:04,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:04,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:09,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:09,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:10,008 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:00:10,011 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:00:10,013 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:00:10,036 INFO Thread-12 :118 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:00:14,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:15,015 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:16,858 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:00:16,860 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 03:00:16,860 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 03:00:16,860 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 03:00:16,861 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 03:00:16,861 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:00:16,861 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:00:16,861 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:00:17,039 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:00:19,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:20,040 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:00:20,430 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:24,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:25,431 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:28,112 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:00:29,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:31,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:32,045 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/config.yaml +2024-03-28 03:00:34,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:36,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:39,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:41,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:44,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:46,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:49,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:51,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:54,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:00:56,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:00:58,113 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:00:59,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:02,114 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:04,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:07,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:09,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:12,116 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:14,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:17,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:19,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:22,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:24,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:27,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:28,114 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:01:28,954 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:01:28,956 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:01:28,956 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:01:28,956 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:01:29,068 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:01:29,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:32,957 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:34,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:35,737 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:01:35,738 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:01:35,739 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:01:35,739 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:01:36,071 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:01:38,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:38,071 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:01:39,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:43,070 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:44,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:48,071 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:49,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:53,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:54,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:01:58,073 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:01:58,115 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:01:59,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:03,116 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:04,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:08,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:09,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:13,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:14,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:18,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:19,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:23,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:24,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:28,116 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:02:29,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:29,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:34,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:34,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:39,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:39,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:44,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:44,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:46,819 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:02:46,820 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:02:46,820 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:02:46,823 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:02:47,100 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:02:49,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:49,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:53,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:02:53,572 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:02:53,572 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:02:53,573 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:02:54,103 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:02:54,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:02:55,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:02:56,104 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:02:58,117 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:02:59,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:01,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:04,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:06,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:09,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:11,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:14,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:16,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:19,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:21,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:24,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:26,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:28,118 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:03:29,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:32,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:34,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:37,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:39,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:42,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:44,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:47,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:49,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:52,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:54,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:03:57,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:03:58,119 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:03:59,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:03,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:04,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:05,941 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:04:05,942 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:04:05,942 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:04:05,943 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:04:06,131 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:04:08,944 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:09,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:12,646 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:04:12,647 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:04:12,647 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:04:12,648 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:04:13,134 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:04:14,134 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:04:14,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:14,885 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:19,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:19,886 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:24,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:24,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:28,120 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:04:29,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:30,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:34,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:35,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:39,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:40,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:44,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:45,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:49,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:50,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:54,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:04:55,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:04:58,121 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:04:59,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:01,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:04,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:06,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:09,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:11,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:14,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:16,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:19,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:21,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:24,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:25,494 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:05:25,496 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:05:25,496 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:05:25,497 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:05:26,162 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:05:26,497 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:28,122 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:05:29,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:32,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:32,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:05:32,223 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:05:32,223 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:05:32,223 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:05:33,166 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:05:34,166 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:05:34,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:37,516 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:39,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:42,517 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:44,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:47,518 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:49,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:52,519 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:54,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:05:57,521 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:05:58,123 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:05:59,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:03,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:04,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:08,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:09,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:13,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:14,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:18,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:19,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:23,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:24,233 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:28,124 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:06:29,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:29,234 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:34,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:34,237 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:39,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:39,238 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:44,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:44,238 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:44,282 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:06:44,284 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:06:44,284 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:06:44,284 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:06:45,193 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:06:49,239 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:49,285 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:50,989 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:06:50,990 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:06:50,991 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:06:50,991 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:06:51,195 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:06:54,196 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:06:54,240 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:06:55,238 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:06:58,125 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:06:59,242 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:01,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:04,242 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:06,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:09,248 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:11,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:14,250 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:16,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:19,251 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:21,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:24,252 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:26,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:28,126 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:07:29,253 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:32,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:34,254 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:37,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:39,255 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:42,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:44,256 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:47,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:49,257 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:52,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:54,261 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:07:57,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:07:58,127 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:07:59,262 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:03,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:03,270 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:08:03,271 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:08:03,272 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:08:03,272 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:08:04,221 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:08:04,263 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:08,273 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:09,265 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:09,983 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:08:09,984 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:08:09,985 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:08:09,985 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:08:10,224 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:08:12,225 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:08:14,241 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:14,266 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:19,242 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:19,267 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:24,243 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:24,268 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:28,128 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:08:29,269 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:30,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:34,270 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:35,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:39,271 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:40,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:44,272 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:45,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:49,273 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:50,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:54,274 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:08:55,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:08:58,129 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:08:59,275 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:01,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:04,276 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:06,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:09,280 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:11,132 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:14,282 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:16,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:19,282 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:21,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:21,758 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:09:21,763 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:09:21,763 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:09:21,764 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:09:22,252 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:09:24,283 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:26,765 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:28,130 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:09:28,477 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:09:28,479 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:09:28,479 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:09:28,480 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:09:29,254 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:09:29,284 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:30,255 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:09:32,083 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:34,285 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:37,084 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:39,286 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:42,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:44,287 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:47,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:49,288 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:52,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:54,289 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:09:57,088 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:09:58,131 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:09:59,291 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:02,132 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:04,292 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:07,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:09,293 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:12,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:14,294 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:17,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:19,295 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:22,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:24,296 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:27,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:28,132 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:10:29,298 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:33,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:34,299 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:38,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:39,300 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:41,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:10:41,221 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:10:41,222 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:10:41,222 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:10:41,281 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:10:43,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:44,301 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:47,940 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:10:47,942 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:10:47,942 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:10:47,943 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:10:48,284 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:10:49,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:49,302 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:50,285 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:10:54,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:10:54,303 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:10:58,134 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:10:59,304 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:00,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:04,305 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:05,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:09,306 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:10,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:14,307 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:15,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:19,308 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:20,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:24,309 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:25,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:28,135 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:11:29,310 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:31,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:34,311 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:36,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:39,312 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:41,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:44,312 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:46,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:49,313 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:51,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:54,314 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:56,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:11:58,135 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:11:59,315 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:11:59,503 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:11:59,504 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:11:59,504 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:11:59,505 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:12:00,312 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:12:01,505 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:04,316 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:06,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:12:06,220 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:12:06,220 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:12:06,221 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:12:06,314 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:12:07,492 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:08,315 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:12:09,317 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:12,492 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:14,318 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:17,493 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:19,319 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:22,494 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:24,320 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:27,495 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:28,137 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:12:29,321 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:33,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:34,322 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:38,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:39,323 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:43,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:44,324 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:48,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:49,325 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:53,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:54,326 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:12:58,138 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:12:59,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:12:59,327 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:04,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:04,328 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:09,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:09,329 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:14,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:14,330 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:16,905 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:13:16,906 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:13:16,906 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:13:16,906 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:13:17,341 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:13:19,331 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:19,908 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:23,630 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:13:23,631 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:13:23,631 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:13:23,632 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:13:24,331 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:24,344 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:13:25,906 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:26,345 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:13:28,139 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:13:29,333 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:31,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:34,333 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:36,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:39,334 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:41,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:44,335 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:46,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:49,336 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:51,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:54,337 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:13:56,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:13:58,140 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:13:59,338 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:02,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:04,339 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:07,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:09,340 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:12,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:14,341 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:17,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:19,342 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:22,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:24,343 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:27,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:28,141 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:14:29,344 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:33,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:34,345 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:35,675 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:14:35,677 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:14:35,677 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:14:35,678 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:14:36,369 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:14:38,678 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:39,346 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:42,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:14:42,404 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:14:42,404 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:14:42,405 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:14:43,372 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:14:44,346 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:44,372 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:14:44,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:49,347 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:49,671 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:54,348 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:14:54,672 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:14:58,142 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:14:59,349 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:00,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:04,350 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:05,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:09,351 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:10,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:14,352 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:15,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:19,353 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:20,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:24,354 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:25,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:28,143 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:15:29,355 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:31,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:34,356 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:36,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:39,357 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:41,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:44,358 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:46,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:49,360 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:51,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:54,361 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:15:54,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:15:54,822 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:15:54,822 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:15:54,823 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:15:55,398 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:15:56,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:15:58,144 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:15:59,361 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:01,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:16:01,532 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:16:01,532 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:16:01,533 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:16:02,400 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:16:02,813 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:04,362 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:04,401 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:16:07,814 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:09,363 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:12,815 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:14,364 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:17,816 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:19,365 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:22,817 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:24,366 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:27,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:28,145 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:16:29,367 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:33,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:34,368 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:38,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:39,369 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:43,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:44,370 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:48,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:49,371 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:53,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:54,372 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:16:58,146 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:16:59,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:16:59,373 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:04,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:04,374 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:09,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:09,375 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:12,430 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:17:12,431 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:17:12,431 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:17:12,432 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:17:13,427 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:17:14,376 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:14,432 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:19,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:17:19,149 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:17:19,149 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:17:19,150 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:17:19,382 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:19,430 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:17:20,384 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:20,430 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:17:24,383 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:25,385 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:28,147 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:17:29,384 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:31,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:34,385 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:36,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:39,385 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:41,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:44,387 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:46,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:49,388 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:51,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:54,389 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:17:56,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:17:58,148 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:17:59,390 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:02,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:04,390 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:07,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:09,391 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:12,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:14,392 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:17,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:19,393 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:22,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:24,394 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:27,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:28,148 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:18:29,395 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:31,878 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:18:31,879 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:18:31,880 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:18:31,880 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:18:32,457 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:18:32,881 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:34,397 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:37,881 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:38,607 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:18:38,609 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:18:38,609 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:18:38,609 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:18:39,398 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:39,460 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:18:40,461 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:18:43,852 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:44,399 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:48,853 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:49,400 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:53,855 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:54,401 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:18:58,149 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:18:59,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:18:59,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:04,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:04,403 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:09,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:09,404 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:14,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:14,405 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:19,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:19,406 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:24,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:24,407 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:28,151 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:19:29,408 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:30,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:34,409 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:35,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:39,410 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:40,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:44,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:45,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:49,412 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:50,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:50,818 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:19:50,819 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:19:50,820 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:19:50,820 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:19:51,487 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:19:54,413 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:19:55,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:19:57,523 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:19:57,524 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:19:57,524 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:19:57,525 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:19:58,152 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:19:58,490 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:19:58,490 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:19:59,414 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:01,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:04,415 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:06,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:09,416 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:11,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:14,417 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:16,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:19,418 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:21,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:24,419 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:26,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:28,152 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:20:29,421 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:32,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:34,422 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:37,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:39,423 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:42,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:44,424 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:47,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:49,425 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:52,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:54,426 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:20:57,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:20:58,153 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:20:59,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:03,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:04,428 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:08,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:09,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:21:09,088 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:21:09,088 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:21:09,089 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:21:09,428 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:09,515 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:21:14,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:14,429 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:15,805 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:21:15,807 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:21:15,807 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:21:15,807 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:21:16,518 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:21:16,519 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:21:19,430 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:20,049 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:24,431 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:25,050 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:28,154 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:21:29,432 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:30,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:34,433 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:35,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:39,434 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:40,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:44,435 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:45,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:49,436 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:50,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:54,437 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:21:55,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:21:58,155 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:21:59,438 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:01,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:04,439 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:06,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:09,440 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:11,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:14,441 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:16,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:19,444 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:21,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:24,445 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:26,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:22:26,046 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:22:26,046 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:22:26,046 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:22:26,544 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:22:27,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:28,156 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:22:29,446 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:32,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:32,747 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:22:32,748 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:22:32,748 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:22:32,749 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:22:33,547 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:22:34,447 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:35,548 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:22:38,034 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:39,448 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:43,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:44,449 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:48,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:49,450 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:53,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:54,451 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:22:58,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:22:58,157 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:22:59,452 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:03,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:04,453 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:08,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:09,454 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:13,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:14,455 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:18,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:19,456 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:23,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:24,457 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:28,158 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:23:29,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:29,458 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:34,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:34,459 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:39,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:39,460 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:43,844 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:23:43,845 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:23:43,846 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:23:43,846 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:23:44,461 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:44,576 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:23:44,847 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:49,462 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:49,848 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:50,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:23:50,550 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:23:50,551 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:23:50,551 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:23:50,578 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:23:53,579 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:23:54,462 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:23:55,786 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:23:58,159 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:23:59,464 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:01,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:04,464 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:06,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:09,465 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:11,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:14,466 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:16,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:19,467 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:21,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:24,468 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:26,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:28,160 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:24:29,470 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:32,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:34,470 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:37,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:39,471 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:42,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:44,472 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:47,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:49,473 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:52,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:54,474 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:24:57,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:24:58,161 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:24:59,475 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:02,475 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:25:02,477 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:25:02,477 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:25:02,477 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:02,478 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:25:02,605 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:25:04,477 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:07,479 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:09,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:25:09,185 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:25:09,185 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:25:09,185 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:25:09,478 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:09,608 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:25:11,609 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:25:13,426 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:14,479 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:18,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:19,480 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:23,428 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:24,481 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:28,162 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:25:29,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:29,482 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:34,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:34,483 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:39,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:39,484 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:44,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:44,485 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:49,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:49,486 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:54,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:25:54,487 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:25:58,163 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:25:59,488 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:00,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:04,489 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:05,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:09,490 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:10,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:14,491 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:15,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:19,492 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:20,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:20,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:26:20,219 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:26:20,219 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:26:20,220 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:26:20,634 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:26:24,493 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:25,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:26,924 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:26:26,925 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:26:26,925 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:26:26,926 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:26:27,637 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:26:28,164 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:26:29,494 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:29,637 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:26:31,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:34,494 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:36,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:39,495 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:41,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:44,496 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:46,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:49,497 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:51,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:54,498 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:26:56,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:26:58,165 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:26:59,499 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:02,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:04,500 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:07,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:09,501 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:12,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:14,502 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:17,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:19,503 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:22,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:24,504 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:27,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:28,166 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:27:29,505 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:33,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:34,506 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:38,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:38,705 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:27:38,706 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:27:38,707 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:27:38,707 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:27:39,507 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:39,664 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:27:43,708 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:44,508 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:45,422 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:27:45,424 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:27:45,424 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:27:45,425 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:27:45,666 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:27:47,667 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:27:49,509 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:49,664 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:54,511 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:27:54,665 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:27:58,167 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:27:59,512 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:00,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:04,513 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:05,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:09,514 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:10,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:14,515 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:15,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:19,515 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:20,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:24,516 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:25,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:28,168 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:28:29,517 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:31,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:34,518 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:36,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:39,519 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:41,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:44,520 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:46,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:49,521 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:51,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:54,522 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:28:56,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:28:56,337 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:28:56,339 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:28:56,339 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:28:56,339 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:28:56,692 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:28:58,169 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:28:59,523 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:02,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:03,061 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:29:03,063 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:29:03,063 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:29:03,064 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:29:03,695 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:29:04,524 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:05,695 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:29:07,300 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:09,525 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:12,301 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:14,526 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:17,302 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:19,528 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:22,302 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:24,529 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:27,303 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:28,170 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:29:29,530 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:33,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:34,530 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:38,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:39,531 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:43,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:44,533 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:48,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:49,534 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:53,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:54,534 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:29:58,171 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:29:59,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:29:59,535 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:04,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:04,536 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:09,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:09,537 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:14,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:14,431 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:30:14,432 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:30:14,432 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:30:14,433 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:30:14,538 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:14,722 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:30:19,434 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:19,539 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:21,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:30:21,140 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:30:21,140 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:30:21,141 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:30:21,724 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:30:23,725 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:30:24,540 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:25,386 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:28,172 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:30:29,541 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:31,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:34,542 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:36,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:39,543 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:41,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:44,544 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:46,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:49,545 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:51,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:54,546 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:30:56,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:30:58,173 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:30:59,547 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:02,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:04,549 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:07,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:09,550 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:12,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:14,551 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:17,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:19,552 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:22,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:24,553 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:27,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:28,174 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:31:29,554 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:33,065 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:31:33,066 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:31:33,067 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:31:33,067 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:33,068 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:31:33,752 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:31:34,555 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:38,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:39,556 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:39,798 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:31:39,799 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:31:39,799 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:31:39,800 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:31:40,754 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:31:41,755 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:31:44,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:44,557 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:49,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:49,558 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:54,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:54,559 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:31:58,175 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:31:59,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:31:59,560 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:04,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:04,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:09,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:09,564 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:14,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:14,565 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:19,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:19,566 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:24,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:24,567 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:28,176 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:32:29,568 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:30,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:34,569 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:35,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:39,570 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:40,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:44,571 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:45,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:49,572 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:50,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:51,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:32:51,221 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:32:51,221 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:32:51,222 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:32:51,782 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:32:54,573 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:55,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:32:57,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:32:57,932 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:32:57,933 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:32:57,933 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:32:58,176 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:32:58,784 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:32:59,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:32:59,785 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:33:00,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:04,575 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:05,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:09,576 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:10,412 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:14,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:15,413 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:19,578 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:20,414 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:24,579 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:25,415 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:28,177 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:33:29,580 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:31,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:34,581 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:36,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:39,582 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:41,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:44,583 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:46,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:49,584 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:51,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:54,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:33:56,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:33:58,178 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:33:59,585 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:02,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:04,586 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:07,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:09,587 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:10,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:34:10,168 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:34:10,168 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:34:10,169 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:34:10,812 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:34:13,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:14,588 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:16,884 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:34:16,885 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:34:16,885 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:34:16,886 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:34:17,815 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:34:19,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:19,589 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:19,816 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:34:24,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:24,590 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:28,179 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:34:29,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:29,591 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:34,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:34,592 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:39,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:39,593 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:44,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:44,594 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:49,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:49,595 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:54,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:54,596 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:34:58,184 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:34:59,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:34:59,597 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:04,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:04,598 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:09,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:09,599 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:14,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:14,600 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:19,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:19,601 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:24,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:24,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:28,181 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:35:28,423 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:35:28,424 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:35:28,424 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:35:28,426 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:35:28,841 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:35:29,427 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:29,602 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:34,428 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:34,603 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:35,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:35:35,158 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:35:35,158 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:35:35,159 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:35:35,844 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:35:37,845 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:35:39,604 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:40,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:44,605 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:45,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:49,610 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:50,403 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:54,611 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:35:55,404 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:35:58,182 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:35:59,613 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:01,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:04,614 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:06,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:09,615 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:11,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:14,616 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:16,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:19,617 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:21,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:24,618 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:26,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:28,185 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:36:29,619 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:32,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:34,620 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:37,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:39,621 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:42,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:44,622 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:47,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:47,689 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:36:47,690 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:36:47,690 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:36:47,691 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:36:47,871 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:36:49,623 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:52,692 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:54,402 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:36:54,404 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:36:54,404 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:36:54,404 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:36:54,630 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:36:54,874 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:36:55,874 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:36:58,184 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:36:58,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:36:59,631 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:03,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:04,632 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:08,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:09,633 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:13,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:14,634 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:18,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:19,635 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:23,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:24,636 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:28,185 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:37:29,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:29,637 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:34,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:34,638 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:39,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:39,639 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:44,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:44,640 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:49,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:49,641 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:54,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:37:54,642 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:37:58,186 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:37:59,643 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:00,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:04,644 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:05,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:05,932 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:38:05,933 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:38:05,933 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:38:05,934 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:38:06,900 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:38:09,645 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:10,935 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:12,645 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:38:12,646 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:38:12,647 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:38:12,647 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:38:12,903 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:38:13,903 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:38:14,646 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:16,893 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:19,647 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:21,894 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:24,648 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:26,895 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:28,187 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:38:29,650 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:32,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:34,650 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:37,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:39,651 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:42,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:44,652 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:47,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:49,653 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:52,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:54,655 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:38:57,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:38:58,188 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:38:59,656 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:03,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:04,659 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:08,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:09,660 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:13,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:14,661 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:18,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:19,662 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:23,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:24,285 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:39:24,286 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:39:24,287 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:39:24,287 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:39:24,663 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:24,932 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:39:28,189 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:39:29,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:29,664 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:31,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:39:31,014 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:39:31,015 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:39:31,015 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:39:31,935 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:39:31,935 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:39:34,253 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:34,665 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:39,254 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:39,666 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:44,255 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:44,667 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:49,256 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:49,669 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:54,257 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:39:54,670 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:39:58,190 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:39:59,671 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:00,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:04,672 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:05,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:09,673 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:10,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:14,674 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:15,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:19,675 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:20,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:24,676 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:25,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:28,191 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:40:29,677 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:31,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:34,678 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:36,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:39,679 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:41,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:42,305 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:40:42,306 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:40:42,307 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:40:42,307 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:40:42,962 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:40:44,680 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:46,308 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:49,031 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:40:49,032 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:40:49,033 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:40:49,033 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:40:49,681 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:49,966 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:40:49,966 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:40:52,287 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:54,682 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:40:57,288 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:40:58,192 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:40:59,683 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:03,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:04,684 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:08,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:09,685 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:13,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:14,686 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:18,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:19,687 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:23,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:24,689 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:28,193 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:41:29,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:29,690 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:34,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:34,691 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:39,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:39,692 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:44,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:44,693 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:49,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:49,694 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:54,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:41:54,695 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:41:58,194 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:41:59,696 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:00,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:01,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:42:01,195 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:42:01,195 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:42:01,195 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:42:01,993 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:42:04,697 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:05,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:07,916 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:42:07,917 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:42:07,917 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:42:07,918 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:42:07,996 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:42:09,698 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:09,997 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:42:11,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:14,699 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:16,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:19,700 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:21,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:24,701 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:26,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:28,195 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:42:29,702 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:31,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:34,703 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:36,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:39,705 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:41,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:44,706 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:46,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:49,706 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:51,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:54,707 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:42:56,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:42:58,196 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:42:59,708 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:02,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:04,709 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:07,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:09,710 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:12,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:14,711 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:17,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:18,774 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:43:18,775 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:43:18,776 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:43:18,776 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:43:19,026 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:43:19,712 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:22,778 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:24,713 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:25,524 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:43:25,525 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:43:25,525 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:43:25,526 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:43:26,029 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:43:28,030 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:43:28,197 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:43:28,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:29,720 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:33,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:34,721 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:38,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:39,721 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:43,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:44,722 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:48,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:49,723 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:53,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:54,724 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:43:58,197 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:43:59,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:43:59,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:04,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:04,726 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:09,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:09,727 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:14,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:14,728 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:19,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:19,729 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:24,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:24,730 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:28,198 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:44:29,732 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:30,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:34,733 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:35,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:36,837 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:44:36,839 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:44:36,839 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:44:36,839 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:44:37,057 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:44:39,734 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:40,840 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:43,561 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:44:43,562 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:44:43,562 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:44:43,563 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:44:44,060 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:44:44,741 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:46,061 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:44:46,800 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:49,741 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:51,801 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:54,742 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:44:56,802 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:44:58,199 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:44:59,743 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:02,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:04,744 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:07,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:09,745 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:12,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:14,746 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:17,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:19,750 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:22,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:24,752 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:27,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:28,200 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:45:29,753 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:33,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:34,753 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:38,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:39,754 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:43,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:44,755 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:48,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:49,756 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:53,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:54,757 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:45:55,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:45:55,123 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:45:55,124 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:45:55,124 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:45:56,087 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:45:58,201 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:45:59,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:45:59,758 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:01,838 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:46:01,839 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:46:01,840 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:46:01,840 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:46:02,089 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:46:04,090 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:46:04,759 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:05,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:09,760 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:10,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:14,761 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:15,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:19,762 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:20,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:24,763 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:25,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:28,202 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:46:29,764 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:30,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:34,766 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:35,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:39,767 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:40,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:44,768 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:45,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:49,769 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:50,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:54,769 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:46:55,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:46:58,203 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:46:59,770 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:01,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:04,771 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:06,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:09,772 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:11,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:14,410 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:47:14,411 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:47:14,411 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:47:14,412 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:47:14,773 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:15,118 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:47:16,413 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:19,774 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:21,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:47:21,152 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:47:21,152 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:47:21,153 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:47:21,416 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:22,121 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:47:23,122 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:47:24,775 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:26,414 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:28,204 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:47:29,776 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:32,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:34,777 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:37,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:39,778 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:42,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:44,779 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:47,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:49,780 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:52,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:54,781 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:47:57,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:47:58,208 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:47:59,782 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:03,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:04,783 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:08,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:09,784 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:13,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:14,785 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:18,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:19,786 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:23,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:24,787 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:28,206 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:48:29,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:29,788 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:32,379 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:48:32,380 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:48:32,380 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:48:32,382 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:48:33,149 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:48:34,384 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:34,789 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:39,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:48:39,114 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:48:39,115 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:48:39,115 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:48:39,151 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:48:39,790 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:40,362 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:41,152 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:48:44,792 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:45,363 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:49,793 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:50,364 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:54,794 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:48:55,365 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:48:58,206 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:48:59,795 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:01,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:04,796 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:06,208 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:09,797 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:11,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:14,799 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:16,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:19,800 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:21,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:24,801 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:26,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:28,209 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:49:29,802 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:32,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:34,807 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:37,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:39,808 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:42,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:44,809 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:47,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:49,650 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:49:49,651 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:49:49,651 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:49:49,652 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:49:49,810 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:50,180 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:49:52,653 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:54,811 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:49:56,375 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:49:56,376 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:49:56,376 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:49:56,377 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:49:57,182 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:49:57,763 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:49:58,209 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:49:59,183 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:49:59,812 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:03,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:04,813 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:08,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:09,814 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:13,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:14,815 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:18,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:19,816 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:23,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:24,820 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:28,209 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:50:29,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:29,821 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:34,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:34,822 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:39,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:39,823 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:44,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:44,824 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:49,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:49,825 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:54,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:50:54,826 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:50:58,210 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:50:59,828 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:00,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:04,829 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:05,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:07,414 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:51:07,415 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:51:07,415 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:51:07,416 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:51:08,210 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:51:09,830 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:10,417 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:14,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:51:14,174 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:51:14,174 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:51:14,175 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:51:14,213 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:51:14,831 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:16,417 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:17,214 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:51:19,832 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:21,418 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:24,833 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:26,418 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:28,211 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:51:29,834 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:32,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:34,835 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:37,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:39,839 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:42,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:44,839 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:47,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:49,841 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:52,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:54,842 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:51:57,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:51:58,212 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:51:59,843 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:03,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:04,844 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:08,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:09,846 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:13,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:14,847 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:18,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:19,848 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:23,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:24,849 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:25,808 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:52:25,809 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:52:25,809 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:52:25,810 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:52:26,240 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:52:28,213 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:52:29,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:29,850 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:32,548 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:52:32,549 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:52:32,549 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:52:32,550 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:52:33,242 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:52:34,796 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:34,851 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:35,243 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:52:39,797 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:39,852 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:44,798 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:44,853 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:49,799 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:49,854 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:54,799 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:52:54,855 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:52:58,214 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:52:59,856 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:00,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:04,857 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:05,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:09,858 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:10,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:14,859 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:15,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:19,860 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:20,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:24,861 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:25,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:28,215 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:53:29,862 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:31,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:34,863 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:36,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:39,865 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:41,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:44,752 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:53:44,753 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:53:44,754 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:53:44,754 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:53:44,866 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:45,269 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:53:46,755 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:49,867 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:51,498 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:53:51,500 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:53:51,500 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:53:51,501 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:53:51,807 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:52,272 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:53:53,273 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:53:54,868 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:53:56,808 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:53:58,216 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:53:59,869 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:02,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:04,870 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:07,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:09,871 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:12,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:14,872 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:17,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:19,873 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:22,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:24,873 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:27,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:28,217 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:54:29,874 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:33,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:34,875 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:38,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:39,876 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:43,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:44,877 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:48,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:49,878 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:53,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:54,884 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:54:58,217 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:54:59,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:54:59,885 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:03,379 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:55:03,380 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:55:03,380 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:55:03,381 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:55:04,301 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:55:04,381 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:04,886 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:09,382 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:09,886 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:10,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:55:10,090 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:55:10,090 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:55:10,090 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:55:10,304 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:55:11,304 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:55:14,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:15,330 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:19,888 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:20,331 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:24,889 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:25,332 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:28,218 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:55:29,890 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:31,219 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:34,891 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:36,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:39,892 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:41,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:44,893 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:46,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:49,894 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:51,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:54,896 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:55:56,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:55:58,219 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:55:59,896 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:02,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:04,897 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:07,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:09,898 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:12,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:14,900 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:17,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:19,901 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:22,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:22,754 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:56:22,755 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:56:22,755 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:56:22,756 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:56:23,333 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:56:24,902 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:27,757 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:28,220 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:56:29,480 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:56:29,481 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:56:29,482 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:56:29,482 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:56:29,903 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:30,336 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:56:31,336 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:56:33,754 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:34,904 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:38,755 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:39,904 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:43,756 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:44,906 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:48,757 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:49,907 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:53,758 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:54,908 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:56:58,222 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:56:59,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:56:59,909 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:04,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:04,910 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:09,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:09,911 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:14,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:14,912 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:19,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:19,913 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:24,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:24,914 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:28,222 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:57:29,915 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:30,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:34,916 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:35,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:39,917 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:40,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:42,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:57:42,037 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:57:42,037 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:57:42,038 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:57:42,365 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:57:44,918 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:46,039 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:48,766 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:57:48,767 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:57:48,768 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:57:48,768 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:57:49,368 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:57:49,919 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:51,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:51,369 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:57:54,920 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:57:56,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:57:58,223 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:57:59,921 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:01,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:04,922 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:06,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:09,923 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:11,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:14,924 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:16,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:19,926 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:21,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:24,927 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:26,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:28,224 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:58:29,928 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:32,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:34,929 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:37,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:39,930 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:42,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:44,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:47,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:49,932 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:52,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:54,933 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:58:57,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:58:58,225 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:58:59,934 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:00,574 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:59:00,575 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:59:00,576 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:59:00,576 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:59:01,396 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:59:02,577 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:04,935 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:07,309 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 03:59:07,310 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 03:59:07,311 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 03:59:07,311 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 03:59:07,398 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 03:59:07,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:09,399 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 03:59:09,936 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:12,725 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:14,937 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:17,726 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:19,938 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:22,727 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:24,939 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:27,728 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:28,226 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:59:29,940 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:33,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:34,941 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:38,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:39,942 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:43,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:44,943 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:48,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:49,944 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:53,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:54,945 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 03:59:58,227 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 03:59:59,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 03:59:59,946 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:04,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:04,947 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:09,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:09,948 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:14,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:14,949 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:19,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 04:00:19,102 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 04:00:19,103 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 04:00:19,103 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 04:00:19,425 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 04:00:19,950 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:20,104 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:24,952 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:25,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:25,838 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 04:00:25,840 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 04:00:25,840 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 04:00:25,841 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 04:00:26,427 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 04:00:27,428 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 04:00:28,228 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 04:00:29,953 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:30,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:34,954 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:35,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:39,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:40,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:44,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:45,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:49,956 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:50,232 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:54,958 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:00:55,233 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:00:58,269 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 04:00:59,959 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:00,270 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:04,960 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:05,271 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:09,960 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:10,272 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:10,921 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 04:01:10,922 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 04:01:10,923 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 04:01:10,923 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 04:01:11,444 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 04:01:14,961 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:15,924 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:17,667 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 04:01:17,669 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 04:01:17,669 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 04:01:17,669 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 04:01:17,925 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 04:01:17,929 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 04:01:17,929 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 04:01:17,929 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 04:01:17,929 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 04:01:17,930 DEBUG SenderThread:118 [sender.py:send():378] send: metric +2024-03-28 04:01:17,930 DEBUG SenderThread:118 [sender.py:send():378] send: history +2024-03-28 04:01:17,930 DEBUG SenderThread:118 [sender.py:send_request():405] send_request: summary_record +2024-03-28 04:01:17,931 INFO SenderThread:118 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-03-28 04:01:17,931 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 04:01:17,932 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 04:01:17,932 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 04:01:17,932 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 04:01:17,932 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 04:01:17,933 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 04:01:17,933 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 04:01:17,951 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 04:01:17,951 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 04:01:17,951 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 04:01:17,951 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 04:01:17,951 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 04:01:17,951 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 04:01:17,952 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 04:01:17,952 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 04:01:17,953 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 04:01:17,953 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 04:01:17,954 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 04:01:17,955 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 04:01:17,956 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 04:01:18,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 04:01:18,130 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 04:01:18,130 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 04:01:18,131 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 04:01:18,131 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 04:01:18,131 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 04:01:18,132 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 04:01:18,132 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 04:01:18,145 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 04:01:18,145 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 04:01:18,145 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 04:01:18,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 04:01:18,146 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 04:01:18,146 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 04:01:18,146 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 04:01:18,147 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 04:01:18,147 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 04:01:18,147 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 04:01:18,149 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 04:01:18,149 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 04:01:18,150 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 04:01:18,447 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/wandb-summary.json +2024-03-28 04:01:19,766 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 04:01:19,766 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 04:01:19,766 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 04:01:19,766 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 04:01:19,767 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 04:01:19,767 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 04:01:19,767 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 04:01:19,768 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 04:01:19,778 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 04:01:19,778 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 04:01:19,778 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 04:01:19,779 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 04:01:19,968 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:21,780 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:24,969 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:26,780 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:29,970 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:31,781 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:34,971 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:36,782 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:39,972 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:41,783 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:44,973 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:46,789 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:48,458 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/config.yaml +2024-03-28 04:01:49,974 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:52,988 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:54,975 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:01:57,989 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:01:59,976 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:02,990 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:04,977 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:07,990 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:09,978 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:12,991 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:14,979 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:17,992 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:19,980 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:22,993 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:24,981 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:27,993 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:29,982 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:32,994 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:34,983 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:37,995 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:39,984 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:42,996 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:44,985 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:47,997 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:49,986 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:52,998 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:54,987 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:02:57,999 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:02:59,989 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:03,000 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:04,990 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:08,001 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:09,991 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:13,002 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:14,992 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:18,003 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:19,993 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:23,004 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:24,994 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:28,004 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:29,994 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:33,005 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:34,995 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:38,006 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:39,996 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:43,007 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:44,997 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:48,008 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:49,999 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:53,009 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:03:55,000 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:03:58,010 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:00,001 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:03,011 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:05,002 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:08,012 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:10,003 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:13,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:15,004 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:18,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:20,005 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:23,014 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:25,007 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:28,016 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:30,008 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:33,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:35,009 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:38,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:40,010 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:43,018 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:45,011 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:48,019 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:50,012 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:53,020 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:04:55,013 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:04:58,021 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:00,014 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:03,022 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:05,015 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:08,022 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:10,016 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:13,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:15,017 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:18,024 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:20,018 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:23,025 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:25,020 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:28,025 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:30,021 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:33,026 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:35,022 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:38,027 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:40,023 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:43,028 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:45,024 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:48,028 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:50,025 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:53,029 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:05:55,026 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:05:58,030 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:00,027 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:03,031 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:05,028 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:08,032 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:10,029 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:13,033 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:15,030 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:18,034 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:20,031 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:23,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:25,032 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:28,036 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:30,033 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:33,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:35,034 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:38,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:40,035 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:43,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:45,037 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:48,039 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:50,038 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:53,040 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:06:55,039 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:06:58,041 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:00,040 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:03,041 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:05,041 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:08,042 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:10,042 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:13,043 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:15,043 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:18,044 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:20,044 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:23,045 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:25,045 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:28,046 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:30,046 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:33,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:35,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:38,047 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:40,048 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:43,048 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:45,049 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:48,049 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:50,050 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:53,050 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:07:55,051 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:07:58,051 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:00,052 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:03,051 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:05,053 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:08,052 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:10,053 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:13,053 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:15,054 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:18,054 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:20,055 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:23,055 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:25,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:28,056 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:30,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:33,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:35,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:38,058 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:40,059 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:43,059 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:45,060 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:48,060 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:50,061 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:53,061 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:08:55,062 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:08:58,062 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:00,063 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:03,063 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:05,064 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:08,064 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:10,065 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:13,065 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:15,066 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:18,066 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:20,067 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:23,067 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:25,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:28,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:30,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:33,068 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:35,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:38,069 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:40,070 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:43,070 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:45,071 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:48,071 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:50,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:53,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:09:55,073 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:09:58,072 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:00,074 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:03,073 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:05,075 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:08,074 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:10,076 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:13,075 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:15,077 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:18,076 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:20,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:23,077 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:25,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:28,078 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:30,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:33,079 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:35,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:38,080 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:40,082 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:43,081 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:45,083 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:48,082 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:50,083 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:53,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:10:55,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:10:58,085 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:00,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:03,086 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:05,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:08,087 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:10,088 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:13,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:15,089 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:18,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:20,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:23,090 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:25,091 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:28,091 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:30,092 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:33,092 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:35,093 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:38,093 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:40,094 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:43,094 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:45,095 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:48,095 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:50,096 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:53,096 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:11:55,097 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:11:58,097 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:00,098 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:03,098 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:05,099 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:08,099 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:10,100 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:13,100 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:15,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:18,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:20,102 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:23,101 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:25,103 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:28,103 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:30,104 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:33,104 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:35,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:38,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:40,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:43,106 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:45,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:48,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:50,108 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:53,108 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:12:55,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:12:58,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:00,110 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:03,110 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:05,111 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:08,111 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:10,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:13,112 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:15,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:18,113 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:20,114 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:23,114 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:25,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:28,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:30,116 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:33,115 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:35,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:38,116 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:40,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:43,117 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:45,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:48,118 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:50,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:53,119 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:13:55,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:13:58,120 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:00,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:03,121 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:05,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:08,122 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:10,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:13,123 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:15,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:18,124 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:20,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:23,125 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:25,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:28,126 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:30,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:33,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:35,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:38,128 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:40,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:43,129 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:45,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:48,130 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:50,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:53,131 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:14:55,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:14:58,132 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:00,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:03,133 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:05,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:08,134 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:10,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:13,135 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:15,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:18,136 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:20,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:23,137 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:25,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:28,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:30,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:33,138 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:35,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:38,139 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:40,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:43,140 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:45,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:48,141 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:50,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:53,142 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:15:55,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:15:58,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:00,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:03,143 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:05,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:08,144 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:10,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:13,145 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:15,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:18,146 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:20,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:23,147 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:25,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:28,148 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:30,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:33,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:35,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:38,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:40,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:43,150 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:45,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:48,151 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:50,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:53,152 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:16:55,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:16:58,153 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:00,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:03,154 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:05,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:08,155 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:10,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:13,156 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:15,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:18,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:20,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:23,157 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:25,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:28,158 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:30,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:33,159 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:35,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:38,160 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:40,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:43,161 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:45,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:48,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:50,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:53,162 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:17:55,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:17:58,163 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:00,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:03,164 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:05,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:08,165 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:10,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:13,166 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:15,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:18,167 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:20,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:23,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:25,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:28,168 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:30,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:33,169 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:35,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:38,170 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:40,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:43,171 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:45,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:48,172 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:50,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:53,173 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:18:55,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:18:58,174 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:00,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:03,175 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:05,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:08,176 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:10,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:13,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:15,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:18,177 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:20,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:23,178 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:25,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:28,179 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:30,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:33,180 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:35,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:38,181 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:40,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:43,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:45,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:48,182 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:50,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:53,183 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:19:55,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:19:58,184 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:00,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:03,185 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:05,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:08,186 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:10,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:13,187 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:15,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:18,188 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:20,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:23,189 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:25,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:28,190 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:30,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:33,191 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:35,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:38,192 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:40,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:43,193 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:45,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:48,194 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:50,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:53,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:20:55,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:20:58,196 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:00,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:03,197 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:05,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:08,198 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:10,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:13,199 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:15,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:18,200 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:20,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:23,201 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:25,209 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:28,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:30,210 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:33,202 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:35,211 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:38,203 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:40,212 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:43,204 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:45,213 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:48,205 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:50,214 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:53,206 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:21:55,215 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:21:58,207 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:00,216 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:03,057 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 04:22:03,057 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 04:22:03,057 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 04:22:03,057 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 04:22:03,058 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 04:22:03,059 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 04:22:03,059 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 04:22:03,060 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 04:22:03,061 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 04:22:03,088 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 04:22:03,088 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 04:22:03,088 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 04:22:03,088 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 04:22:03,088 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 04:22:03,089 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 04:22:03,089 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 04:22:03,091 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 04:22:03,100 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 04:22:03,100 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 04:22:03,100 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 04:22:03,101 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 04:22:04,102 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:05,217 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:09,103 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:10,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:14,104 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:15,220 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:19,105 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:20,221 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:24,106 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:25,222 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:29,107 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:30,223 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:34,108 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:35,224 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:39,109 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:40,225 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:44,110 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:45,226 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:45,411 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 04:22:45,412 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 04:22:45,412 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 04:22:45,412 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 04:22:45,412 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 04:22:45,413 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 04:22:45,415 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 04:22:45,416 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 04:22:45,417 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 04:22:46,969 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 04:22:48,970 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 04:22:49,127 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:50,227 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:54,149 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:22:55,228 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:22:58,974 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 04:22:59,195 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:23:00,229 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:23:00,974 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 04:23:02,975 INFO Thread-12 :118 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240328_025827-lnd0ipu3/files/output.log +2024-03-28 04:23:04,218 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:23:05,230 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:23:09,240 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:23:10,231 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive +2024-03-28 04:23:10,941 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: pause +2024-03-28 04:23:10,941 INFO HandlerThread:118 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-03-28 04:23:10,941 INFO HandlerThread:118 [system_monitor.py:finish():203] Stopping system monitor +2024-03-28 04:23:10,941 DEBUG SystemMonitor:118 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-28 04:23:10,941 DEBUG SystemMonitor:118 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-28 04:23:10,941 INFO HandlerThread:118 [interfaces.py:finish():202] Joined cpu monitor +2024-03-28 04:23:10,942 DEBUG SystemMonitor:118 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-28 04:23:10,942 INFO HandlerThread:118 [interfaces.py:finish():202] Joined disk monitor +2024-03-28 04:23:10,955 INFO HandlerThread:118 [interfaces.py:finish():202] Joined gpu monitor +2024-03-28 04:23:10,955 INFO HandlerThread:118 [interfaces.py:finish():202] Joined memory monitor +2024-03-28 04:23:10,955 INFO HandlerThread:118 [interfaces.py:finish():202] Joined network monitor +2024-03-28 04:23:10,955 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: resume +2024-03-28 04:23:10,955 INFO HandlerThread:118 [handler.py:handle_request_resume():699] starting system metrics thread +2024-03-28 04:23:10,955 INFO HandlerThread:118 [system_monitor.py:start():194] Starting system monitor +2024-03-28 04:23:10,956 INFO SystemMonitor:118 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-28 04:23:10,956 DEBUG SenderThread:118 [sender.py:send():378] send: stats +2024-03-28 04:23:10,956 INFO SystemMonitor:118 [interfaces.py:start():190] Started cpu monitoring +2024-03-28 04:23:10,957 INFO SystemMonitor:118 [interfaces.py:start():190] Started disk monitoring +2024-03-28 04:23:10,958 INFO SystemMonitor:118 [interfaces.py:start():190] Started gpu monitoring +2024-03-28 04:23:10,958 INFO SystemMonitor:118 [interfaces.py:start():190] Started memory monitoring +2024-03-28 04:23:10,959 INFO SystemMonitor:118 [interfaces.py:start():190] Started network monitoring +2024-03-28 04:23:14,958 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 04:23:15,232 DEBUG HandlerThread:118 [handler.py:handle_request():146] handle_request: keepalive