diff --git "a/wandb/run-20240404_092718-5h7d1ksf/logs/debug-internal.log" "b/wandb/run-20240404_092718-5h7d1ksf/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240404_092718-5h7d1ksf/logs/debug-internal.log" @@ -0,0 +1,1706 @@ +2024-04-04 09:27:18,864 INFO StreamThr :121 [internal.py:wandb_internal():86] W&B internal server running at pid: 121, started at: 2024-04-04 09:27:18.863707 +2024-04-04 09:27:18,866 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status +2024-04-04 09:27:19,519 INFO WriterThread:121 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/run-5h7d1ksf.wandb +2024-04-04 09:27:19,520 DEBUG SenderThread:121 [sender.py:send():378] send: header +2024-04-04 09:27:19,524 DEBUG SenderThread:121 [sender.py:send():378] send: run +2024-04-04 09:27:19,916 INFO SenderThread:121 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files +2024-04-04 09:27:19,917 INFO SenderThread:121 [sender.py:_start_run_threads():1099] run started: 5h7d1ksf with start time 1712222838.864514 +2024-04-04 09:27:19,921 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: check_version +2024-04-04 09:27:19,921 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: check_version +2024-04-04 09:27:19,975 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: run_start +2024-04-04 09:27:19,988 DEBUG HandlerThread:121 [system_info.py:__init__():26] System info init +2024-04-04 09:27:19,988 DEBUG HandlerThread:121 [system_info.py:__init__():41] System info init done +2024-04-04 09:27:19,988 INFO HandlerThread:121 [system_monitor.py:start():194] Starting system monitor +2024-04-04 09:27:19,988 INFO SystemMonitor:121 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-04 09:27:19,988 INFO HandlerThread:121 [system_monitor.py:probe():214] Collecting system info +2024-04-04 09:27:19,989 INFO SystemMonitor:121 [interfaces.py:start():190] Started cpu monitoring +2024-04-04 09:27:19,990 INFO SystemMonitor:121 [interfaces.py:start():190] Started disk monitoring +2024-04-04 09:27:19,990 INFO SystemMonitor:121 [interfaces.py:start():190] Started gpu monitoring +2024-04-04 09:27:19,991 INFO SystemMonitor:121 [interfaces.py:start():190] Started memory monitoring +2024-04-04 09:27:19,992 INFO SystemMonitor:121 [interfaces.py:start():190] Started network monitoring +2024-04-04 09:27:20,008 DEBUG HandlerThread:121 [system_info.py:probe():150] Probing system +2024-04-04 09:27:20,010 DEBUG HandlerThread:121 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-04 09:27:20,010 DEBUG HandlerThread:121 [system_info.py:probe():198] Probing system done +2024-04-04 09:27:20,010 DEBUG HandlerThread:121 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-04T09:27:20.008294', 'startedAt': '2024-04-04T09:27:18.857241', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '3634ac9905cc', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.148, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.148, 'min': 0.0, 'max': 0.0}, {'current': 2000.148, 'min': 0.0, 'max': 0.0}, {'current': 2000.148, 'min': 0.0, 'max': 0.0}, {'current': 2000.148, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5533.6725730896}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-04-04 09:27:20,010 INFO HandlerThread:121 [system_monitor.py:probe():224] Finished collecting system info +2024-04-04 09:27:20,010 INFO HandlerThread:121 [system_monitor.py:probe():227] Publishing system info +2024-04-04 09:27:20,010 DEBUG HandlerThread:121 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-04 09:27:20,918 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/conda-environment.yaml +2024-04-04 09:28:13,932 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/conda-environment.yaml +2024-04-04 09:28:14,375 DEBUG HandlerThread:121 [system_info.py:_save_conda():219] Saving conda packages done +2024-04-04 09:28:14,376 INFO HandlerThread:121 [system_monitor.py:probe():229] Finished publishing system info +2024-04-04 09:28:14,386 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,386 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,386 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,388 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,388 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: python_packages +2024-04-04 09:28:14,388 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: stop_status +2024-04-04 09:28:14,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-04 09:28:14,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,390 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,390 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,390 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:14,390 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:14,390 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: python_packages +2024-04-04 09:28:14,393 DEBUG SenderThread:121 [sender.py:send():378] send: telemetry +2024-04-04 09:28:14,393 DEBUG SenderThread:121 [sender.py:send():378] send: config +2024-04-04 09:28:14,394 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: stop_status +2024-04-04 09:28:14,606 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:14,606 DEBUG SenderThread:121 [sender.py:send():378] send: telemetry +2024-04-04 09:28:14,607 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:14,607 WARNING SenderThread:121 [sender.py:send_metric():1316] Seen metric with glob (shouldn't happen) +2024-04-04 09:28:14,607 DEBUG SenderThread:121 [sender.py:send():378] send: telemetry +2024-04-04 09:28:14,607 DEBUG SenderThread:121 [sender.py:send():378] send: files +2024-04-04 09:28:14,607 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-metadata.json with policy now +2024-04-04 09:28:14,933 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/requirements.txt +2024-04-04 09:28:14,933 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:28:14,933 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-metadata.json +2024-04-04 09:28:15,230 INFO wandb-upload_0:121 [upload_job.py:push():131] Uploaded file /tmp/tmpznzxqg8ewandb/316g1mh1-wandb-metadata.json +2024-04-04 09:28:15,969 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:16,933 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:28:19,614 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:19,935 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/config.yaml +2024-04-04 09:28:19,992 DEBUG SystemMonitor:121 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-04 09:28:19,993 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:28:20,970 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:24,994 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:25,971 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:29,995 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:30,972 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:34,996 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:35,973 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:39,997 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:40,460 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:28:40,462 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:40,463 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:40,463 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:40,463 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:40,463 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:28:40,463 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:28:40,465 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:28:40,943 INFO Thread-12 :121 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:28:40,974 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:45,467 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:45,607 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:28:45,609 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:45,609 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:45,609 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:45,610 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 09:28:45,610 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:28:45,610 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:28:45,610 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:28:45,945 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:28:45,975 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:46,945 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:28:49,994 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:28:50,976 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:51,000 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:28:51,947 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/config.yaml +2024-04-04 09:28:55,977 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:28:56,208 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:00,978 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:01,209 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:05,979 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:06,210 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:10,980 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:11,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:15,981 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:16,212 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:19,995 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:29:20,982 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:21,996 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:25,983 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:26,997 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:30,984 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:31,998 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:33,244 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:29:33,246 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:29:33,246 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:29:33,248 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:29:33,963 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:29:35,985 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:37,249 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:38,414 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:29:38,415 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:29:38,415 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:29:38,415 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:29:38,965 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:29:40,965 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:29:40,986 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:42,542 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:45,987 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:47,543 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:49,996 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:29:50,988 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:52,997 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:29:55,989 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:29:57,997 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:00,990 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:02,998 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:05,991 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:07,999 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:10,992 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:13,000 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:15,993 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:18,001 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:19,997 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:30:20,994 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:23,998 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:25,995 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:26,856 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:30:26,857 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:30:26,857 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:30:26,857 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:30:26,982 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:30:29,858 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:30,996 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:31,992 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:30:31,994 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:30:31,994 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:30:31,996 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:30:32,984 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:30:32,985 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:30:35,122 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:36,002 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:40,123 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:41,005 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:45,123 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:46,006 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:49,997 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:30:50,998 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:51,007 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:30:55,999 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:30:56,008 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:01,000 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:01,009 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:06,000 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:06,010 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:11,001 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:11,011 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:16,002 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:16,012 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:19,998 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:31:20,052 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:31:20,054 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:31:20,054 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:31:20,054 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:31:21,002 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:31:21,013 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:21,055 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:25,202 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:31:25,204 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:31:25,204 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:31:25,205 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:31:26,004 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:31:26,014 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:26,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:27,005 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:31:31,015 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:31,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:36,016 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:36,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:41,017 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:41,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:46,018 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:46,332 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:49,999 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:31:51,019 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:52,001 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:31:56,020 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:31:57,002 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:01,021 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:02,003 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:06,022 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:07,004 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:11,023 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:12,005 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:13,155 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:32:13,156 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:32:13,157 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:32:13,159 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:32:14,022 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:32:16,023 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:17,160 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:18,281 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:32:18,282 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:32:18,282 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:32:18,283 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:32:19,024 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:32:20,000 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:32:21,024 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:21,025 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:32:23,001 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:26,025 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:28,002 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:31,026 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:33,003 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:36,027 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:38,004 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:41,028 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:43,005 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:46,029 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:48,006 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:50,001 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:32:51,031 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:54,002 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:32:56,032 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:32:59,003 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:01,032 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:04,004 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:05,424 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:33:05,425 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:33:05,425 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:33:05,426 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:33:06,033 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:06,041 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:33:09,427 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:10,561 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:33:10,562 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:33:10,562 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:33:10,563 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:33:11,034 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:11,043 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:33:13,044 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:33:14,687 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:16,035 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:19,688 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:20,002 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:33:21,036 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:25,003 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:26,037 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:30,004 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:31,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:35,005 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:36,039 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:40,006 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:41,040 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:45,007 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:46,042 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:50,003 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:33:51,004 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:51,043 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:56,005 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:33:56,044 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:33:58,853 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:33:58,855 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:33:58,855 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:33:58,855 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:33:59,061 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:34:01,045 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:01,856 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:04,004 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:34:04,005 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:34:04,006 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:34:04,006 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:34:04,063 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:34:05,063 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:34:06,046 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:07,128 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:11,047 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:12,129 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:16,048 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:17,130 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:20,004 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:34:21,049 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:23,005 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:26,050 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:28,006 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:31,051 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:33,007 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:36,052 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:38,008 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:41,053 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:43,009 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:46,054 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:48,010 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:50,005 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:34:51,055 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:51,392 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:34:51,393 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:34:51,394 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:34:51,394 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:34:52,081 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:34:53,395 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:56,056 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:34:56,526 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:34:56,527 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:34:56,528 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:34:56,528 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:34:57,083 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:34:58,663 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:34:59,084 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:35:01,057 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:03,664 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:06,058 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:08,665 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:11,062 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:13,666 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:16,063 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:18,667 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:20,006 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:35:21,064 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:24,007 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:26,065 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:29,008 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:31,066 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:34,009 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:36,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:39,010 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:41,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:42,892 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:35:42,893 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:35:42,894 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:35:42,894 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:35:43,100 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:35:44,895 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:46,069 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:48,030 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:35:48,031 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:35:48,032 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:35:48,032 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:35:48,102 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:35:49,103 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:35:50,007 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:35:50,009 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:51,070 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:35:55,008 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:35:56,071 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:00,010 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:01,072 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:05,010 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:06,073 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:10,011 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:11,074 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:15,012 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:16,075 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:20,008 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:36:21,009 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:21,076 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:26,010 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:26,077 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:31,010 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:31,078 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:35,356 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:36:35,357 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:36:35,358 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:36:35,358 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:36:36,079 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:36,120 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:36:36,359 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:40,496 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:36:40,497 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:36:40,498 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:36:40,498 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:36:41,080 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:41,122 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:36:41,629 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:42,123 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:36:46,081 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:46,630 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:50,009 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:36:51,082 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:52,009 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:36:56,083 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:36:57,011 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:01,084 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:02,012 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:06,085 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:07,012 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:11,086 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:12,013 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:16,087 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:17,014 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:20,010 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:37:21,088 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:23,011 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:26,089 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:28,012 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:28,406 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:37:28,407 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:37:28,407 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:37:28,408 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:37:29,139 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:37:31,090 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:33,409 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:33,554 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:37:33,555 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:37:33,555 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:37:33,556 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:37:34,140 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:37:36,091 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:36,141 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:37:38,680 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:41,092 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:43,681 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:46,094 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:48,682 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:50,011 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:37:51,095 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:54,012 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:37:56,096 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:37:59,013 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:01,097 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:04,014 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:06,098 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:09,015 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:11,098 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:14,016 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:16,099 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:19,017 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:20,012 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:38:21,100 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:21,895 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:38:21,897 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:38:21,897 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:38:21,898 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:38:22,157 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:38:24,898 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:26,101 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:27,024 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:38:27,025 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:38:27,025 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:38:27,026 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:38:27,158 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:38:30,158 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:30,159 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:38:31,105 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:35,159 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:36,106 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:40,160 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:41,107 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:45,160 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:46,108 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:50,012 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:38:51,013 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:51,110 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:38:56,014 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:38:56,111 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:01,015 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:01,112 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:06,016 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:06,113 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:11,017 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:11,114 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:14,373 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:39:14,374 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:39:14,375 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:39:14,375 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:39:15,175 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:39:16,115 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:16,376 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:19,505 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:39:19,507 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:39:19,507 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:39:19,507 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:39:20,013 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:39:20,177 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:39:21,116 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:22,014 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:22,178 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:39:26,118 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:27,016 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:31,119 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:32,016 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:36,120 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:37,017 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:41,121 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:42,018 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:46,122 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:47,019 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:50,014 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:39:51,123 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:53,015 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:39:56,124 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:39:58,016 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:01,125 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:03,017 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:06,126 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:07,293 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:40:07,294 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:40:07,295 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:40:07,295 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:40:08,195 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:40:08,296 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:11,127 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:12,435 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:40:12,436 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:40:12,437 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:40:12,437 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:40:13,197 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:40:13,561 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:14,198 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:40:16,128 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:18,562 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:20,015 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:40:21,129 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:24,016 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:26,130 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:29,017 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:31,131 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:34,018 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:36,132 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:39,019 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:41,133 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:44,020 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:46,134 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:49,021 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:50,016 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:40:51,135 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:40:55,018 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:40:56,136 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:00,019 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:01,137 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:01,332 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:41:01,333 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:41:01,333 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:41:01,333 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:41:02,214 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:41:05,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:06,139 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:06,465 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:41:06,467 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:41:06,467 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:41:06,467 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:41:07,216 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:41:08,216 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:41:10,614 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:11,140 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:15,615 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:16,141 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:20,017 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:41:21,018 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:21,142 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:26,019 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:26,143 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:31,020 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:31,144 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:36,021 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:36,145 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:41,022 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:41,147 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:46,023 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:46,148 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:50,018 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:41:51,149 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:52,019 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:54,500 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:41:54,501 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:41:54,502 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:41:54,502 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:41:55,234 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:41:56,150 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:41:57,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:41:59,634 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:41:59,635 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:41:59,635 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:41:59,636 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:42:00,235 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:42:01,151 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:02,236 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:42:02,758 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:06,153 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:07,759 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:11,154 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:12,760 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:16,155 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:17,761 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:20,019 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:42:21,156 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:23,021 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:26,157 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:28,022 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:31,158 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:33,022 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:36,159 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:38,024 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:41,160 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:43,024 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:46,161 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:47,370 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:42:47,371 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:42:47,371 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:42:47,372 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:42:48,253 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:42:48,372 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:50,020 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:42:51,162 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:52,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:42:52,515 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:42:52,515 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:42:52,516 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:42:53,255 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:42:53,643 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:42:54,255 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:42:56,163 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:42:58,644 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:01,164 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:03,645 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:06,165 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:08,645 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:11,166 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:13,647 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:16,167 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:18,647 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:20,021 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:43:21,168 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:24,023 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:26,169 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:29,024 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:31,170 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:34,024 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:36,171 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:39,025 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:40,501 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:43:40,503 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:43:40,503 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:43:40,503 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:43:41,172 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:41,273 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:43:44,504 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:45,640 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:43:45,641 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:43:45,641 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:43:45,642 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:43:46,173 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:46,276 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:43:48,277 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:43:49,772 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:50,022 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:43:51,174 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:43:55,024 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:43:56,175 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:00,025 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:01,176 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:05,026 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:06,177 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:10,027 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:11,178 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:15,028 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:16,179 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:20,023 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:44:21,024 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:21,180 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:26,025 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:26,181 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:31,026 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:31,182 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:33,057 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:44:33,058 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:44:33,058 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:44:33,059 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:44:33,324 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:44:36,060 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:36,183 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:38,198 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:44:38,200 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:44:38,200 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:44:38,200 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:44:38,326 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:44:40,326 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:44:41,184 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:41,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:46,185 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:46,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:50,024 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:44:51,186 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:52,025 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:44:56,187 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:44:57,026 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:01,188 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:02,027 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:06,189 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:07,028 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:11,190 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:12,029 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:16,191 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:17,030 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:20,025 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:45:21,192 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:23,026 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:26,193 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:26,455 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:45:26,456 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:45:26,457 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:45:26,458 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:45:27,344 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:45:28,459 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:31,194 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:31,602 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:45:31,603 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:45:31,604 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:45:31,604 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:45:32,345 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:45:33,727 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:34,346 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:45:36,195 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:38,727 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:41,196 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:43,728 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:46,197 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:48,729 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:50,026 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:45:51,198 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:54,027 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:45:56,199 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:45:59,028 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:01,200 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:04,029 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:06,201 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:09,030 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:11,202 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:14,031 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:16,204 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:19,032 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:19,549 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:46:19,550 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:46:19,550 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:46:19,550 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:46:20,027 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:46:20,362 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:46:21,205 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:24,684 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:46:24,685 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:46:24,685 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:46:24,686 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:24,686 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:46:25,364 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:46:26,206 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:26,364 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:46:29,812 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:31,207 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:34,813 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:36,208 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:39,814 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:41,209 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:44,815 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:46,210 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:49,816 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:50,027 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:46:51,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:46:55,029 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:46:56,212 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:00,030 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:01,213 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:05,031 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:06,214 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:10,032 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:11,215 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:12,402 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:47:12,403 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:47:12,403 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:47:12,404 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:47:13,381 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:47:15,404 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:16,216 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:17,529 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:47:17,530 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:47:17,531 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:47:17,531 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:47:18,383 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:47:20,028 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:47:20,384 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:47:21,029 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:21,218 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:26,030 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:26,218 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:31,031 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:31,219 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:36,032 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:36,221 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:41,033 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:41,221 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:46,034 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:46,223 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:50,068 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:47:51,069 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:51,224 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:47:56,070 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:47:56,225 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:01,071 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:01,226 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:05,729 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:48:05,730 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:48:05,731 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:48:05,731 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:48:06,227 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:06,401 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:48:06,732 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:10,873 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:48:10,874 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:48:10,874 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:48:10,875 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:48:11,228 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:11,403 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:48:12,004 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:12,404 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:48:16,229 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:17,005 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:20,030 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:48:21,230 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:22,031 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:26,231 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:27,032 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:31,232 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:32,033 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:36,233 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:37,034 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:41,234 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:42,035 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:46,235 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:47,035 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:50,032 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:48:51,236 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:53,034 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:48:56,237 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:48:57,469 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:48:57,470 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:48:57,471 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:48:57,471 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:48:58,420 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:48:58,471 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:01,238 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:02,601 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:49:02,602 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:49:02,603 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:49:02,603 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:49:03,422 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:49:03,733 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:04,423 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:49:06,239 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:08,734 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:11,240 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:13,735 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:16,241 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:18,736 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:20,032 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:49:21,242 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:24,033 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:26,243 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:29,034 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:31,244 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:34,035 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:36,245 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:39,036 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:41,246 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:44,037 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:46,247 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:49,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:50,021 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:49:50,022 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:49:50,022 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:49:50,023 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:49:50,033 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:49:50,439 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:49:51,248 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:55,034 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:49:55,160 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:49:55,161 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:49:55,161 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:49:55,162 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:49:55,441 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:49:56,254 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:49:56,441 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:50:00,288 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:01,255 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:05,289 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:06,256 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:10,290 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:11,257 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:15,290 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:16,258 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:20,034 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:50:21,034 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:21,259 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:26,035 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:26,261 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:31,037 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:31,262 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:36,037 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:36,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:41,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:41,264 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:42,975 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:50:42,976 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:50:42,976 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:50:42,977 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:50:43,458 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:50:46,265 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:46,978 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:48,106 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:50:48,108 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:50:48,108 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:50:48,109 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:50:48,460 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:50:50,035 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:50:50,461 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:50:51,266 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:52,036 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:50:56,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:50:57,037 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:01,268 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:02,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:06,269 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:07,039 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:11,270 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:12,040 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:16,273 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:17,041 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:20,036 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:51:21,274 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:23,037 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:26,275 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:28,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:31,276 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:33,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:35,813 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:51:35,814 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:51:35,814 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:51:35,814 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:51:36,277 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:36,478 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:51:38,815 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:40,956 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:51:40,957 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:51:40,958 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:51:40,958 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:51:41,278 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:41,480 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:51:42,480 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:51:44,083 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:46,279 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:49,084 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:50,037 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:51:51,280 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:51:55,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:51:56,281 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:00,039 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:01,282 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:05,040 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:06,282 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:10,041 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:11,283 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:15,041 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:16,284 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:20,038 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:52:21,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:21,285 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:26,039 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:26,286 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:28,602 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:52:28,604 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:52:28,604 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:52:28,604 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:52:29,497 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:52:31,287 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:31,605 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:33,733 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:52:33,734 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:52:33,735 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:52:33,735 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:52:34,499 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:52:36,288 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:36,500 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:52:36,859 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:41,288 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:41,860 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:46,289 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:46,861 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:50,039 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:52:51,290 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:52,040 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:52:56,291 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:52:57,040 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:01,292 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:02,042 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:06,293 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:07,042 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:11,294 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:12,044 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:16,295 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:17,045 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:20,039 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:53:21,296 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:21,573 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:53:21,574 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:53:21,574 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:53:21,575 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:53:22,517 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:53:22,575 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:26,297 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:26,713 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:53:26,715 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:53:26,715 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:53:26,715 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:53:27,519 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:53:27,843 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:28,520 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:53:31,298 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:32,844 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:36,299 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:37,845 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:41,301 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:42,846 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:46,302 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:47,847 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:50,040 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:53:51,303 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:53,042 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:53:56,304 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:53:58,043 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:01,304 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:03,044 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:06,305 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:08,044 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:11,306 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:13,045 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:15,260 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:54:15,261 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:54:15,262 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:54:15,262 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:54:15,537 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:54:16,307 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:18,263 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:20,041 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:54:20,391 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:54:20,393 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:54:20,393 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:54:20,394 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:54:20,538 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:54:21,309 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:22,539 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:54:23,521 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:26,310 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:28,524 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:31,311 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:33,524 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:36,312 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:38,526 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:41,313 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:43,526 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:46,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:48,527 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:50,042 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:54:51,315 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:54,044 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:54:56,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:54:59,044 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:01,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:04,046 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:06,318 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:08,183 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:55:08,185 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:55:08,185 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:55:08,185 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:55:08,556 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:55:09,186 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:11,319 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:13,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:55:13,325 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:55:13,326 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:55:13,326 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:55:13,557 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:55:14,454 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:14,558 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:55:16,320 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:19,455 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:20,043 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:55:21,321 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:25,045 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:26,322 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:30,046 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:31,323 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:35,046 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:36,324 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:40,047 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:41,325 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:45,048 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:46,326 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:50,044 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:55:51,045 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:51,327 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:55:56,046 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:55:56,328 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:00,567 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:56:00,568 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:56:00,569 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:56:00,569 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:56:00,574 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:56:01,329 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:01,570 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:05,698 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:56:05,699 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:56:05,699 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:56:05,700 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:56:06,330 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:06,577 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:56:06,577 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:56:06,867 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:11,331 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:11,868 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:16,333 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:16,869 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:20,045 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:56:21,333 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:22,046 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:26,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:27,047 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:31,335 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:32,048 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:36,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:37,049 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:41,337 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:42,050 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:46,339 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:47,051 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:50,046 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:56:51,340 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:53,047 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:54,034 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:56:54,035 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:56:54,035 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:56:54,036 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:56:54,594 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:56:56,341 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:56:59,037 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:56:59,180 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:56:59,181 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:56:59,181 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:56:59,182 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:56:59,596 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:57:00,596 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:57:01,342 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:04,314 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:06,343 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:09,315 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:11,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:14,316 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:16,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:19,317 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:20,047 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:57:21,346 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:25,048 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:26,347 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:30,049 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:31,348 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:35,050 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:36,349 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:40,051 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:41,350 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:45,052 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:46,351 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:47,120 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:57:47,121 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:57:47,121 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:57:47,122 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:57:47,614 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:57:50,048 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:57:51,049 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:57:51,352 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:52,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:57:52,254 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:57:52,254 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:57:52,255 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:57:52,616 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:57:54,617 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:57:56,358 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:57:56,382 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:01,359 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:01,383 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:06,360 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:06,383 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:11,361 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:11,384 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:16,362 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:16,385 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:20,049 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:58:21,363 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:22,050 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:26,364 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:27,051 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:31,365 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:32,052 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:36,366 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:37,053 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:39,242 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:58:39,243 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:58:39,243 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:58:39,244 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:58:39,633 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:58:41,367 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:42,244 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:44,379 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:58:44,381 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:58:44,381 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:58:44,381 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:58:44,635 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:58:46,369 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:46,636 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:58:47,508 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:50,058 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:58:51,370 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:53,059 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:58:56,371 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:58:58,061 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:01,372 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:03,061 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:06,373 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:08,062 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:11,373 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:13,063 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:16,374 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:18,064 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:20,051 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:59:21,375 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:24,053 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:26,376 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:29,053 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:31,377 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:33,069 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:59:33,070 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:59:33,071 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:59:33,073 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:59:33,653 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:59:34,074 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:36,378 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:38,211 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 09:59:38,212 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 09:59:38,212 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 09:59:38,213 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 09:59:38,654 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 09:59:39,344 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:40,655 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 09:59:41,379 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:44,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:46,380 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:49,345 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:50,052 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 09:59:51,381 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 09:59:55,054 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 09:59:56,382 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:00,054 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:01,383 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:05,056 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:06,384 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:10,057 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:11,385 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:15,057 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:16,386 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:20,055 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:00:21,056 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:21,387 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:25,901 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:00:25,902 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:00:25,902 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:00:25,903 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:00:26,388 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:26,672 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:00:26,903 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:31,046 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:00:31,048 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:00:31,048 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:00:31,049 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:00:31,389 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:31,674 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:00:32,174 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:32,674 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:00:36,390 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:37,175 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:41,390 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:42,176 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:46,394 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:47,177 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:50,054 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:00:51,395 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:53,055 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:00:56,396 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:00:58,057 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:01,396 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:03,058 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:06,398 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:08,058 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:11,399 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:13,059 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:16,400 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:18,060 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:18,977 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:01:18,978 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:01:18,978 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:01:18,979 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:01:19,692 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:01:20,055 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:01:21,401 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:24,056 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:24,120 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:01:24,121 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:01:24,122 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:01:24,122 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:01:24,694 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:01:25,694 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:01:26,402 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:29,249 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:31,403 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:34,250 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:36,404 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:39,251 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:41,405 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:44,252 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:46,405 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:49,253 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:50,056 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:01:51,407 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:01:55,058 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:01:56,407 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:00,059 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:01,408 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:05,060 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:06,410 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:10,061 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:11,410 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:11,799 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:02:11,801 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:02:11,801 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:02:11,801 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:02:12,713 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:02:15,802 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:16,411 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:16,944 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:02:16,946 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:02:16,946 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:02:16,946 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:02:17,715 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:02:19,715 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:02:20,057 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:02:21,058 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:21,412 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:26,059 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:26,413 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:31,060 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:31,414 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:36,061 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:36,415 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:41,062 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:41,416 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:46,063 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:46,417 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:50,059 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:02:51,420 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:52,059 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:02:56,421 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:02:57,060 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:01,422 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:02,061 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:04,992 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:03:04,993 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:03:04,994 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:03:04,994 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:03:05,732 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:03:06,423 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:07,995 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:10,134 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:03:10,135 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:03:10,135 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:03:10,136 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:03:10,734 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:03:11,424 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:11,735 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:03:13,267 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:16,428 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:18,268 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:20,059 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:03:21,428 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:24,061 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:26,429 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:29,062 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:31,430 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:34,062 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:36,431 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:39,063 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:41,432 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:44,064 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:46,433 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:49,065 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:50,060 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:03:51,434 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:55,062 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:03:56,435 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:03:58,298 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:03:58,299 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:03:58,299 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:03:58,300 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:03:58,753 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:04:00,300 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:01,436 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:03,446 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:04:03,448 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:04:03,448 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:04:03,449 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:04:03,755 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:04:05,610 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:05,756 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:04:06,437 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:10,611 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:11,439 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:15,612 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:16,440 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:20,061 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:04:21,062 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:21,441 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:26,063 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:26,442 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:31,064 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:31,443 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:36,065 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:36,444 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:41,066 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:41,445 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:46,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:46,446 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:50,062 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:04:51,059 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:04:51,060 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:04:51,061 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:04:51,061 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:04:51,447 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:51,773 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:04:52,062 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:56,201 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:04:56,202 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:04:56,202 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:04:56,202 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:04:56,448 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:04:56,776 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:04:57,333 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:04:57,776 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:05:01,449 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:02,334 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:06,450 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:07,335 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:11,452 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:12,336 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:16,453 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:17,338 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:20,063 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:05:21,454 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:23,064 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:26,456 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:28,065 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:31,457 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:33,066 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:36,458 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:38,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:41,459 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:43,068 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:44,428 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:05:44,430 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:05:44,430 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:05:44,430 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:05:44,794 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:05:46,460 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:48,431 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:49,566 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:05:49,567 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:05:49,567 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:05:49,568 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:05:49,796 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:05:50,064 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:05:51,461 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:51,797 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:05:54,065 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:05:56,462 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:05:59,066 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:01,463 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:04,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:06,464 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:09,068 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:11,465 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:14,069 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:16,466 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:19,070 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:20,065 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:06:21,467 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:25,066 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:26,469 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:30,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:31,470 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:35,068 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:36,471 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:36,966 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:06:36,967 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:06:36,968 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:06:36,968 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:06:37,817 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:06:40,969 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:41,472 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:42,107 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:06:42,108 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:06:42,109 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:06:42,109 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:06:42,819 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:06:43,820 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:06:46,237 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:46,473 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:50,066 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:06:51,474 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:52,067 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:06:56,475 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:06:57,068 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:01,477 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:02,069 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:06,478 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:07,070 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:11,479 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:12,071 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:16,480 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:17,072 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:20,067 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:07:21,481 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:23,068 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:26,482 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:28,069 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:29,872 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:07:29,873 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:07:29,874 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:07:29,874 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:07:30,837 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:07:31,483 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:33,875 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:35,011 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:07:35,012 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:07:35,013 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:07:35,013 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:07:35,839 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:07:36,484 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:37,840 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:07:39,140 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:41,485 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:44,141 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:46,486 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:49,142 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:50,067 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:07:51,487 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:07:55,069 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:07:56,488 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:00,070 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:01,489 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:05,071 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:06,490 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:10,072 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:11,492 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:15,073 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:16,493 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:20,068 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:08:21,069 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:21,494 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:22,103 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:08:22,105 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:08:22,105 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:08:22,105 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:08:22,856 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:08:26,106 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:26,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:27,241 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:08:27,242 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:08:27,243 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:08:27,243 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:08:27,858 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:08:29,859 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:08:31,372 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:31,495 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:36,373 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:36,497 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:41,374 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:41,498 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:46,375 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:46,503 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:50,069 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:08:51,505 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:52,071 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:08:56,506 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:08:57,072 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:01,507 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:02,072 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:06,508 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:07,073 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:11,509 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:12,074 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:15,038 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:09:15,039 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:09:15,040 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:09:15,040 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:09:15,876 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:09:16,510 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:18,041 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:20,070 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:09:20,169 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:09:20,171 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:09:20,171 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:09:20,171 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:09:20,877 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:09:21,511 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:21,878 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/output.log +2024-04-04 10:09:23,298 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:26,512 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:28,299 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:31,513 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:33,299 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:36,514 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:38,300 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:41,515 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:43,301 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:46,517 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:48,302 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:50,071 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:09:50,548 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:09:50,549 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:09:50,549 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:09:50,549 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:09:50,889 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:09:51,518 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:53,550 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report +2024-04-04 10:09:55,696 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:09:55,697 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:09:55,698 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:09:55,698 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:09:55,848 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: partial_history +2024-04-04 10:09:55,850 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 10:09:55,851 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 10:09:55,851 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 10:09:55,851 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 10:09:55,851 DEBUG SenderThread:121 [sender.py:send():378] send: metric +2024-04-04 10:09:55,851 DEBUG SenderThread:121 [sender.py:send():378] send: history +2024-04-04 10:09:55,851 DEBUG SenderThread:121 [sender.py:send_request():405] send_request: summary_record +2024-04-04 10:09:55,852 INFO SenderThread:121 [sender.py:_save_file():1365] saving file wandb-summary.json with policy end +2024-04-04 10:09:55,853 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: pause +2024-04-04 10:09:55,853 INFO HandlerThread:121 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-04 10:09:55,853 INFO HandlerThread:121 [system_monitor.py:finish():203] Stopping system monitor +2024-04-04 10:09:55,854 DEBUG SystemMonitor:121 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-04 10:09:55,854 INFO HandlerThread:121 [interfaces.py:finish():202] Joined cpu monitor +2024-04-04 10:09:55,854 DEBUG SystemMonitor:121 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-04 10:09:55,854 INFO HandlerThread:121 [interfaces.py:finish():202] Joined disk monitor +2024-04-04 10:09:55,865 INFO HandlerThread:121 [interfaces.py:finish():202] Joined gpu monitor +2024-04-04 10:09:55,865 INFO HandlerThread:121 [interfaces.py:finish():202] Joined memory monitor +2024-04-04 10:09:55,865 INFO HandlerThread:121 [interfaces.py:finish():202] Joined network monitor +2024-04-04 10:09:55,866 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: resume +2024-04-04 10:09:55,866 INFO HandlerThread:121 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-04 10:09:55,866 INFO HandlerThread:121 [system_monitor.py:start():194] Starting system monitor +2024-04-04 10:09:55,866 INFO SystemMonitor:121 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-04 10:09:55,866 INFO SystemMonitor:121 [interfaces.py:start():190] Started cpu monitoring +2024-04-04 10:09:55,867 INFO SystemMonitor:121 [interfaces.py:start():190] Started disk monitoring +2024-04-04 10:09:55,867 INFO SystemMonitor:121 [interfaces.py:start():190] Started gpu monitoring +2024-04-04 10:09:55,868 INFO SystemMonitor:121 [interfaces.py:start():190] Started memory monitoring +2024-04-04 10:09:55,869 INFO SystemMonitor:121 [interfaces.py:start():190] Started network monitoring +2024-04-04 10:09:55,871 DEBUG SenderThread:121 [sender.py:send():378] send: stats +2024-04-04 10:09:55,891 INFO Thread-12 :121 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240404_092718-5h7d1ksf/files/wandb-summary.json +2024-04-04 10:09:56,519 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: keepalive +2024-04-04 10:09:58,872 DEBUG HandlerThread:121 [handler.py:handle_request():146] handle_request: status_report