diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -27179,3 +27179,5424 @@ 2024-03-28 00:15:00,746 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report 2024-03-28 00:15:02,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported 2024-03-28 00:15:02,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:04,212 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:04,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:04,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:06,112 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:06,212 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:07,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:07,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:08,213 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:09,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:09,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:11,463 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:11,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:15:11,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:15:11,519 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:15:12,215 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:12,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:12,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:14,216 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:14,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:14,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:16,217 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:16,835 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:17,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:17,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:20,085 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:15:20,219 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:21,504 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:15:21,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:21,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:22,196 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:22,220 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:23,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:23,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:24,220 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:26,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:26,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:26,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:15:26,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:15:26,519 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:15:27,732 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:28,222 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:28,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:28,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:30,223 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:31,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:31,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:32,224 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:32,908 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:33,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:33,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:36,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:36,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:36,225 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:38,226 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:38,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:38,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:38,278 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:40,226 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:40,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:40,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:41,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:15:41,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:15:41,519 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:15:43,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:43,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:43,746 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:44,228 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:45,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:45,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:46,229 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:48,230 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:48,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:48,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:48,997 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:50,087 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:15:50,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:50,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:52,231 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:52,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:52,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:54,232 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:54,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:15:55,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:55,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:56,233 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:15:56,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:15:56,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:15:56,519 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:15:58,742 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:15:58,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:58,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:15:59,732 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:00,234 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:01,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:01,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:02,235 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:03,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:16:03,121 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:16:03,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:16:03,122 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:16:03,235 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:16:03,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:03,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:04,236 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:05,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:05,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:05,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:08,237 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:08,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:08,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:10,238 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:10,461 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:10,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:10,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:11,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:16:11,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:16:11,519 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:16:12,238 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:13,879 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:13,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:15,826 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:15,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:15,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:16,240 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:18,241 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:18,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:19,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:20,088 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:16:20,241 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:21,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:21,046 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:21,090 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:24,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:24,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:24,243 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:26,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:26,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:26,243 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:26,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:16:26,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:16:26,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:16:26,738 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:29,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:29,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:30,245 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:31,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:31,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:31,932 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:32,245 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:34,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:34,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:34,246 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:37,302 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:37,591 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:16:37,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:37,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:38,248 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:39,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:39,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:40,249 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:41,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:16:41,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:16:41,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:16:42,250 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:42,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:42,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:42,730 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:44,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:44,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:46,252 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:47,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:47,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:48,026 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:48,253 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:49,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:49,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:50,090 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:16:50,253 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:52,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:52,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:53,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:54,255 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:54,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:54,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:56,255 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:56,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:16:56,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:16:56,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:16:57,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:57,652 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:58,256 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:16:58,760 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:16:59,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:16:59,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:02,258 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:02,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:02,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:04,259 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:04,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:04,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:06,260 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:07,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:07,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:09,478 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:09,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:09,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:10,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:17:10,188 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:17:10,188 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:17:10,189 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:17:10,262 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:17:10,262 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:11,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:17:11,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:17:11,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:17:12,262 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:12,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:12,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:14,263 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:14,850 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:16,589 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:17:16,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:16,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:18,264 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:19,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:19,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:20,091 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:17:20,093 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:20,265 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:21,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:21,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:22,266 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:23,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:23,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:25,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:26,267 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:26,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:17:26,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:17:26,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:17:26,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:26,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:28,268 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:28,718 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:28,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:30,269 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:30,949 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:31,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:31,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:33,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:33,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:34,270 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:36,271 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:36,315 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:36,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:36,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:38,272 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:38,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:38,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:41,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:17:41,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:17:41,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:17:41,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:41,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:41,750 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:42,274 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:43,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:43,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:44,275 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:46,275 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:46,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:46,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:47,056 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:48,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:48,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:50,093 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:17:50,277 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:51,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:51,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:52,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:52,437 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:54,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:17:55,453 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:17:55,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:55,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:56,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:17:56,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:17:56,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:17:57,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:57,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:17:57,671 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:17:58,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:00,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:00,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:00,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:02,281 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:02,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:02,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:03,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:05,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:05,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:06,282 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:07,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:07,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:08,283 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:08,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:10,284 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:10,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:10,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:11,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:18:11,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:18:11,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:18:12,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:12,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:13,898 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:14,285 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:15,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:15,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:16,286 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:17,283 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:18:17,285 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:18:17,285 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:18:17,285 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:18:17,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:18:17,737 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:17,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:18,287 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:19,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:20,094 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:18:21,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:21,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:22,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:23,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:23,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:24,289 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:24,608 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:26,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:26,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:26,289 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:26,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:18:26,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:18:26,520 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:18:28,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:28,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:29,963 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:30,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:30,105 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:30,290 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:32,291 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:33,293 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:18:33,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:33,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:34,292 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:35,316 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:36,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:36,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:38,293 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:38,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:38,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:40,294 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:40,424 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:41,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:41,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:41,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:18:41,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:18:41,521 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:18:43,217 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:43,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:45,739 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:45,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:45,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:47,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:47,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:50,096 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:18:50,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:50,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:51,098 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:52,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:52,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:54,298 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:55,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:55,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:56,221 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:18:56,298 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:18:56,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:18:56,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:18:56,521 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:18:57,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:18:57,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:00,299 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:00,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:00,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:01,579 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:02,300 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:02,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:02,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:04,300 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:05,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:05,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:06,938 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:07,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:07,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:08,302 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:10,303 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:11,472 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:19:11,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:11,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:11,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:19:11,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:19:11,521 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:19:12,303 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:12,711 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:14,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:14,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:16,305 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:16,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:16,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:18,305 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:18,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:19,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:19,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:20,098 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:19:20,306 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:21,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:21,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:24,001 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:24,307 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:24,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:24,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:26,308 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:26,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:26,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:26,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:19:26,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:19:26,521 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:19:28,308 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:29,350 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:29,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:29,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:31,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:31,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:32,310 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:34,311 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:34,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:34,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:34,697 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:35,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:19:35,404 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:19:35,404 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:19:35,405 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:19:36,311 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:19:36,312 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:36,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:36,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:39,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:39,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:40,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:40,312 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:41,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:41,414 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:41,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:19:41,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:19:41,521 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:19:42,313 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:44,313 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:44,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:44,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:45,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:46,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:46,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:48,315 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:49,948 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:19:49,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:49,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:50,099 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:19:50,315 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:51,101 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:52,316 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:53,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:53,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:55,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:55,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:56,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:19:56,317 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:19:56,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:19:56,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:19:56,521 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:19:58,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:58,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:19:58,318 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:00,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:00,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:00,318 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:01,498 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:03,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:03,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:04,319 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:05,206 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:05,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:06,320 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:06,846 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:08,321 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:08,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:08,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:10,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:10,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:11,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:20:11,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:20:11,521 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:20:12,322 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:12,702 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:13,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:13,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:14,323 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:15,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:15,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:18,241 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:18,325 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:18,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:18,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:20,101 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:20:20,325 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:20,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:20,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:22,326 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:22,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:22,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:23,594 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:25,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:25,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:26,327 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:26,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:20:26,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:20:26,522 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:20:28,328 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:28,713 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:28,834 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:20:28,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:28,865 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:30,329 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:30,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:30,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:33,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:33,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:34,300 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:34,330 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:35,989 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:35,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:36,330 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:38,331 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:39,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:39,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:39,656 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:41,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:41,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:41,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:20:41,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:20:41,522 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:20:42,332 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:42,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:20:42,363 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:20:42,363 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:20:42,363 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:20:43,333 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:20:44,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:44,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:44,333 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:45,015 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:46,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:46,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:46,333 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:49,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:49,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:50,103 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:20:50,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:50,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:51,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:51,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:52,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:53,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:53,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:54,336 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:20:55,716 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:20:56,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:56,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:56,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:20:56,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:20:56,522 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:20:58,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:58,309 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:20:58,337 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:00,337 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:01,070 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:01,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:01,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:02,338 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:03,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:03,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:06,339 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:06,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:06,815 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:21:06,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:06,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:08,340 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:09,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:09,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:10,340 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:11,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:21:11,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:21:11,522 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:21:11,747 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:11,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:11,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:14,341 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:14,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:14,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:16,342 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:16,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:16,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:17,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:18,343 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:19,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:19,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:20,104 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:21:21,884 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:21,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:22,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:22,488 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:24,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:24,989 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:25,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:26,345 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:26,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:21:26,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:21:26,522 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:21:27,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:27,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:27,700 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:29,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:29,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:30,346 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:32,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:32,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:32,347 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:33,210 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:34,347 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:34,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:34,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:36,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:36,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:38,348 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:38,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:39,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:39,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:40,349 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:41,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:21:41,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:21:41,522 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:21:42,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:42,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:42,350 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:43,671 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:45,451 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:21:45,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:45,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:46,351 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:48,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:48,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:48,352 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:49,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:49,302 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:21:49,303 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:21:49,303 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:21:49,304 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:21:49,352 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:21:50,106 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:21:50,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:50,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:50,352 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:53,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:53,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:54,354 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:54,625 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:21:55,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:55,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:56,354 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:56,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:21:56,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:21:56,523 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:21:58,355 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:21:58,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:58,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:21:59,990 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:00,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:00,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:02,356 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:03,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:03,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:04,356 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:05,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:05,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:05,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:06,357 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:08,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:08,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:10,358 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:10,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:10,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:10,692 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:11,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:22:11,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:22:11,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:22:12,359 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:13,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:13,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:14,360 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:15,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:15,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:16,060 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:17,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:17,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:18,361 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:20,108 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:22:20,361 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:20,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:20,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:21,109 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:22,362 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:23,965 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:22:23,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:24,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:26,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:26,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:26,364 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:26,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:22:26,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:22:26,523 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:22:26,657 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:28,364 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:28,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:28,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:30,364 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:30,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:30,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:32,117 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:33,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:33,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:34,366 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:35,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:35,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:36,366 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:37,469 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:38,367 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:38,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:38,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:40,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:40,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:41,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:22:41,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:22:41,523 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:22:42,368 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:42,673 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:43,001 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:43,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:44,369 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:45,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:45,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:46,369 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:47,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:47,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:48,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:50,109 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:22:50,370 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:51,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:51,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:52,371 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:53,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:53,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:53,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:22:54,372 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:56,230 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:22:56,231 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:22:56,231 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:22:56,231 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:22:56,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:56,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:56,373 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:22:56,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:22:56,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:22:56,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:22:58,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:58,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:22:58,373 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:22:58,713 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:00,374 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:01,682 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:23:01,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:01,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:02,375 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:04,230 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:04,657 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:04,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:06,376 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:06,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:06,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:08,377 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:09,579 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:09,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:09,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:10,378 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:11,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:23:11,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:23:11,523 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:23:11,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:11,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:14,379 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:14,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:14,657 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:14,925 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:16,379 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:16,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:16,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:18,380 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:19,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:19,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:20,111 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:23:20,113 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:21,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:21,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:22,381 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:24,382 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:24,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:24,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:25,638 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:26,383 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:26,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:23:26,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:23:26,523 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:23:26,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:26,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:29,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:29,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:30,384 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:30,996 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:31,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:31,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:32,385 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:34,386 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:34,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:34,935 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:36,339 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:36,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:36,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:38,387 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:40,336 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:23:40,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:40,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:40,388 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:41,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:23:41,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:23:41,523 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:23:41,749 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:42,389 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:43,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:43,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:45,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:45,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:46,391 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:47,070 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:48,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:48,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:48,391 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:50,112 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:23:50,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:50,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:50,392 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:52,114 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:53,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:53,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:54,393 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:55,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:55,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:56,394 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:23:56,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:23:56,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:23:56,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:23:57,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:23:58,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:58,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:23:58,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:00,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:00,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:02,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:03,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:03,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:24:03,165 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:24:03,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:24:03,166 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:24:03,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:03,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:03,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:24:04,397 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:05,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:05,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:06,397 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:08,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:08,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:08,507 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:10,399 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:10,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:10,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:11,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:24:11,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:24:11,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:24:12,400 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:13,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:13,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:13,712 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:14,400 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:15,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:15,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:18,402 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:18,985 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:24:19,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:19,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:19,216 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:20,114 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:24:20,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:21,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:21,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:22,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:23,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:23,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:24,572 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:26,405 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:26,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:24:26,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:24:26,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:24:26,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:26,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:28,406 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:28,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:28,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:29,926 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:31,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:31,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:32,408 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:33,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:33,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:34,409 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:35,285 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:36,409 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:36,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:36,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:38,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:38,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:40,411 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:40,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:41,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:24:41,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:24:41,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:24:42,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:42,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:42,411 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:44,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:44,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:44,412 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:45,989 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:47,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:47,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:48,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:49,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:49,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:50,115 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:24:50,414 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:51,116 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:52,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:52,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:52,415 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:54,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:54,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:56,416 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:24:56,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:24:56,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:24:56,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:24:56,735 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:24:57,546 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:24:57,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:57,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:24:58,417 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:00,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:00,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:00,418 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:02,063 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:02,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:02,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:04,419 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:05,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:05,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:06,420 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:07,422 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:07,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:07,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:08,421 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:10,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:25:10,123 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:25:10,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:25:10,123 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:25:10,422 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:25:10,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:10,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:11,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:25:11,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:25:11,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:25:12,422 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:12,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:12,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:12,660 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:14,423 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:15,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:15,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:16,424 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:17,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:17,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:18,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:20,117 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:25:20,425 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:20,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:20,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:22,426 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:22,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:22,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:23,471 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:24,427 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:25,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:25,625 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:26,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:25:26,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:25:26,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:25:27,643 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:27,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:28,429 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:28,682 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:30,429 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:30,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:30,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:32,430 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:32,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:32,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:34,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:35,991 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:25:36,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:36,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:36,432 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:38,433 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:38,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:38,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:39,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:40,433 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:40,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:40,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:41,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:25:41,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:25:41,525 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:25:44,028 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:44,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:44,435 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:44,906 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:46,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:46,073 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:46,435 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:48,436 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:49,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:49,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:50,119 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:25:50,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:51,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:51,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:52,438 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:54,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:54,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:54,438 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:55,614 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:25:56,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:56,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:56,439 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:25:56,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:25:56,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:25:56,525 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:25:59,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:25:59,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:00,441 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:00,976 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:01,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:01,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:02,442 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:04,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:04,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:04,443 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:06,327 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:06,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:06,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:08,445 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:09,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:09,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:10,445 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:11,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:11,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:26:11,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:26:11,525 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:26:11,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:11,681 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:12,446 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:15,047 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:26:15,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:15,076 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:16,447 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:17,043 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:17,069 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:26:17,071 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:26:17,071 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:26:17,071 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:26:17,448 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:26:17,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:17,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:18,448 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:20,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:20,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:20,121 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:26:20,448 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:22,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:23,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:23,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:24,450 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:25,090 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:25,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:26,451 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:26,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:26:26,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:26:26,525 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:26:27,678 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:28,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:28,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:28,452 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:30,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:30,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:32,453 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:33,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:33,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:33,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:34,454 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:35,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:35,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:36,455 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:38,483 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:38,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:38,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:40,456 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:40,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:40,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:41,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:26:41,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:26:41,525 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:26:42,457 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:43,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:43,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:43,676 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:44,457 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:45,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:45,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:48,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:48,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:48,459 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:49,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:50,122 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:26:50,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:50,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:50,460 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:52,461 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:53,875 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:26:53,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:53,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:54,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:26:56,463 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:56,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:26:56,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:26:56,525 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:26:56,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:56,866 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:58,463 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:26:58,884 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:58,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:26:59,894 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:00,464 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:01,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:01,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:03,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:03,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:04,465 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:05,267 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:06,466 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:07,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:07,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:08,466 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:09,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:09,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:10,639 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:11,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:27:11,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:27:11,525 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:27:12,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:12,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:12,468 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:14,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:14,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:14,469 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:15,988 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:16,469 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:17,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:17,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:19,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:19,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:20,124 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:27:20,471 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:21,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:22,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:22,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:22,471 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:24,042 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:27:24,044 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:27:24,044 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:27:24,044 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:27:24,204 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:24,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:24,472 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:27:24,473 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:26,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:26,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:26,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:27:26,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:27:26,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:27:26,688 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:28,473 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:29,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:29,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:30,474 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:32,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:32,475 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:32,583 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:27:32,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:32,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:34,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:34,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:36,476 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:37,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:37,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:37,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:38,477 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:39,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:39,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:40,477 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:41,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:27:41,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:27:41,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:27:41,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:41,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:42,682 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:44,479 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:44,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:44,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:46,480 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:46,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:46,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:48,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:48,481 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:49,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:49,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:50,126 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:27:51,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:51,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:52,482 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:53,483 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:27:53,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:53,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:54,483 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:56,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:56,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:56,484 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:27:56,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:27:56,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:27:56,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:27:58,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:58,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:27:58,687 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:00,486 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:00,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:00,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:02,486 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:03,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:03,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:04,212 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:05,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:05,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:06,488 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:08,488 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:08,804 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:28:08,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:08,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:09,582 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:10,490 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:10,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:10,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:11,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:28:11,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:28:11,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:28:13,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:13,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:14,491 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:14,942 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:15,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:15,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:16,492 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:17,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:17,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:18,492 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:20,128 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:28:20,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:20,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:20,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:22,494 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:22,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:22,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:24,495 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:25,648 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:25,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:25,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:26,495 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:26,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:28:26,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:28:26,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:28:27,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:27,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:30,497 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:30,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:30,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:30,988 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:31,014 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:28:31,015 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:28:31,016 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:28:31,016 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:28:31,497 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:28:32,497 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:32,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:33,001 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:33,498 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:34,498 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:35,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:35,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:36,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:37,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:37,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:38,499 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:40,500 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:40,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:40,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:41,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:28:41,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:28:41,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:28:41,653 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:42,501 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:42,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:42,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:44,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:28:45,963 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:28:45,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:45,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:47,115 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:48,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:48,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:50,130 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:28:50,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:50,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:52,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:53,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:53,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:55,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:55,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:56,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:28:56,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:28:56,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:28:57,740 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:28:58,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:28:58,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:00,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:00,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:00,508 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:02,509 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:02,971 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:03,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:03,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:04,510 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:05,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:05,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:07,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:07,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:08,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:08,511 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:09,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:10,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:10,512 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:11,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:29:11,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:29:11,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:29:12,513 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:13,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:13,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:13,741 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:15,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:15,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:16,514 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:18,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:18,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:18,515 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:18,994 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:20,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:20,131 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:29:20,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:20,516 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:23,645 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:29:23,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:23,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:24,332 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:24,517 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:26,518 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:26,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:29:26,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:29:26,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:29:26,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:26,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:28,519 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:28,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:28,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:29,670 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:31,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:31,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:32,521 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:33,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:33,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:34,522 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:35,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:36,522 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:36,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:36,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:38,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:38,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:40,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:40,525 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:41,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:29:41,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:29:41,526 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:29:41,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:41,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:42,526 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:43,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:43,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:45,528 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:45,685 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:45,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:45,737 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:48,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:48,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:49,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:29:49,056 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:29:49,057 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:29:49,057 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:29:49,529 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:29:49,529 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:50,133 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:29:50,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:50,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:51,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:51,530 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:53,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:53,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:53,530 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:55,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:55,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:56,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:29:56,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:29:56,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:29:56,527 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:29:57,532 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:29:58,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:58,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:29:59,533 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:01,534 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:01,697 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:02,112 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:30:02,130 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:02,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:05,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:05,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:05,535 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:07,043 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:07,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:07,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:07,536 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:09,537 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:10,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:10,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:11,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:30:11,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:30:11,527 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:30:12,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:12,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:12,743 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:13,539 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:15,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:15,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:15,539 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:17,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:17,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:17,540 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:18,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:20,135 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:30:20,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:20,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:21,542 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:22,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:22,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:23,542 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:23,724 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:24,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:24,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:25,543 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:26,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:30:26,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:30:26,527 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:30:27,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:27,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:29,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:29,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:29,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:29,545 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:31,545 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:32,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:32,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:33,546 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:34,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:34,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:34,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:37,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:37,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:37,548 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:39,549 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:39,742 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:40,810 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:30:40,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:40,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:41,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:30:41,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:30:41,527 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:30:41,549 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:42,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:42,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:45,088 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:45,551 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:45,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:45,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:47,551 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:47,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:47,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:49,552 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:50,137 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:30:50,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:50,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:50,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:52,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:52,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:53,554 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:55,555 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:55,788 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:30:55,814 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:30:55,815 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:30:55,816 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:30:55,816 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:30:56,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:56,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:56,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:30:56,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:30:56,527 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:30:56,555 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:30:57,555 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:30:58,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:30:58,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:01,105 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:01,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:01,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:01,556 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:03,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:03,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:03,557 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:05,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:05,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:05,558 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:06,485 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:08,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:08,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:09,559 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:10,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:10,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:11,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:31:11,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:31:11,527 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:31:11,560 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:11,743 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:12,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:12,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:13,561 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:15,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:15,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:17,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:17,562 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:18,658 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:31:18,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:18,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:19,563 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:20,139 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:31:20,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:20,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:21,564 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:22,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:23,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:23,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:25,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:25,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:25,565 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:26,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:31:26,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:31:26,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:31:27,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:27,710 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:28,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:28,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:29,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:30,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:30,625 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:33,225 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:33,568 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:33,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:33,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:35,568 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:35,652 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:35,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:37,569 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:38,565 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:38,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:38,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:40,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:40,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:41,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:31:41,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:31:41,527 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:31:41,571 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:43,572 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:43,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:43,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:43,733 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:45,572 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:45,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:45,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:48,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:48,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:49,272 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:49,573 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:50,143 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:31:50,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:50,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:51,574 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:53,575 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:53,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:53,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:54,620 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:31:56,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:31:56,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:31:56,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:31:57,165 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:31:57,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:57,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:31:57,576 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:59,577 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:31:59,972 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:00,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:00,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:01,578 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:02,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:02,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:02,678 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:32:02,679 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:32:02,680 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:32:02,680 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:32:03,578 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:32:05,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:05,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:05,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:05,579 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:07,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:07,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:07,580 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:09,581 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:10,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:10,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:10,685 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:11,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:32:11,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:32:11,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:32:12,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:12,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:13,582 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:15,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:15,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:15,583 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:16,042 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:17,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:17,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:17,584 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:20,142 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:32:20,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:20,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:21,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:21,585 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:22,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:22,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:23,586 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:24,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:24,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:25,587 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:26,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:32:26,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:32:26,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:32:26,690 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:27,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:27,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:29,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:29,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:29,588 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:31,589 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:32,090 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:32,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:32,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:33,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:36,077 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:32:36,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:36,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:37,441 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:37,591 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:38,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:38,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:39,592 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:40,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:40,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:41,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:32:41,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:32:41,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:32:41,593 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:42,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:42,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:42,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:45,595 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:45,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:45,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:47,595 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:47,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:47,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:48,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:49,596 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:50,144 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:32:50,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:50,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:52,777 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:52,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:53,494 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:53,597 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:55,598 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:55,851 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:55,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:56,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:32:56,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:32:56,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:32:57,599 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:32:57,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:57,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:58,691 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:32:59,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:32:59,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:01,600 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:02,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:02,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:03,601 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:04,230 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:04,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:04,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:05,602 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:07,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:07,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:09,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:09,603 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:09,607 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:33:09,609 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:33:09,609 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:33:09,610 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:33:09,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:09,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:10,603 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:33:11,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:33:11,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:33:11,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:33:11,604 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:13,232 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:33:13,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:13,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:13,606 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:14,931 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:16,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:16,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:17,608 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:18,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:18,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:19,608 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:20,146 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:33:20,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:21,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:21,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:21,609 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:23,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:23,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:25,610 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:25,651 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:26,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:26,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:26,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:33:26,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:33:26,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:33:27,611 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:28,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:28,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:29,612 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:31,012 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:31,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:31,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:33,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:33,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:33,613 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:35,614 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:36,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:36,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:36,367 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:37,614 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:38,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:38,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:41,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:41,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:41,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:33:41,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:33:41,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:33:41,616 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:41,708 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:43,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:43,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:43,616 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:45,617 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:46,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:46,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:47,063 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:48,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:48,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:49,618 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:50,147 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:33:51,619 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:51,774 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:33:51,800 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:51,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:52,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:53,620 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:54,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:54,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:56,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:33:56,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:33:56,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:33:56,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:56,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:57,622 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:57,759 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:33:59,623 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:33:59,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:33:59,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:01,624 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:01,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:01,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:03,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:04,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:04,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:05,625 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:06,793 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:06,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:07,626 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:08,486 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:09,628 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:09,693 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:09,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:11,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:34:11,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:34:11,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:34:11,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:11,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:13,629 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:13,757 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:14,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:14,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:15,630 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:16,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:34:16,523 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:34:16,524 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:34:16,524 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:34:16,630 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:34:16,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:16,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:17,631 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:19,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:19,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:20,149 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:34:21,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:21,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:21,633 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:23,633 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:24,467 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:24,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:24,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:25,634 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:26,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:26,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:26,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:34:26,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:34:26,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:34:29,636 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:29,836 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:34:29,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:29,866 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:29,884 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:31,637 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:32,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:32,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:33,638 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:34,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:34,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:35,241 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:37,639 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:37,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:37,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:39,640 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:39,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:39,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:40,594 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:41,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:34:41,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:34:41,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:34:41,641 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:42,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:42,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:44,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:44,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:45,642 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:45,950 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:47,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:47,643 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:47,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:49,644 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:49,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:49,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:50,150 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:34:51,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:52,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:52,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:53,645 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:54,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:54,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:55,646 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:56,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:34:56,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:34:56,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:34:56,719 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:34:57,647 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:34:57,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:34:57,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:00,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:00,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:01,648 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:02,021 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:02,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:02,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:03,650 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:04,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:04,793 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:05,650 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:07,369 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:08,405 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:35:08,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:08,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:09,652 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:11,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:11,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:11,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:35:11,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:35:11,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:35:11,653 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:12,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:13,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:13,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:13,654 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:16,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:16,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:17,655 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:18,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:18,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:18,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:19,655 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:20,152 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:35:21,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:21,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:21,656 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:23,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:23,432 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:35:23,433 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:35:23,434 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:35:23,434 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:35:23,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:23,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:23,657 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:35:25,658 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:26,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:35:26,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:35:26,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:35:26,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:26,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:27,658 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:28,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:28,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:28,754 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:29,659 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:31,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:31,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:33,661 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:33,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:33,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:35,662 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:36,595 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:36,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:37,663 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:38,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:38,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:39,465 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:41,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:35:41,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:35:41,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:35:41,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:41,664 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:41,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:43,665 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:43,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:43,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:44,807 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:45,666 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:47,143 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:35:47,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:47,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:49,667 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:50,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:50,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:50,154 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:35:50,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:51,668 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:52,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:52,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:55,073 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:55,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:55,503 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:35:55,669 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:56,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:35:56,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:35:56,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:35:57,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:57,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:35:57,670 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:35:59,671 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:00,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:00,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:00,857 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:02,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:02,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:03,672 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:05,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:05,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:05,673 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:06,209 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:07,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:07,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:07,674 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:09,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:10,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:11,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:36:11,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:36:11,529 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:36:11,675 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:11,727 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:12,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:12,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:13,676 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:15,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:15,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:15,677 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:16,913 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:17,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:17,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:19,678 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:20,155 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:36:20,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:20,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:21,679 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:22,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:22,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:22,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:23,680 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:25,722 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:36:25,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:25,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:26,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:36:26,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:36:26,530 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:36:27,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:27,682 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:28,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:28,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:29,682 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:30,304 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:36:30,306 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:36:30,306 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:36:30,307 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:36:30,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:30,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:30,683 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:36:31,683 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:32,946 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:33,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:33,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:35,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:35,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:35,684 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:37,685 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:38,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:38,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:38,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:39,686 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:40,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:40,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:41,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:36:41,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:36:41,530 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:36:43,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:43,688 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:43,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:43,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:45,688 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:45,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:45,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:47,689 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:48,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:48,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:48,984 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:50,157 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:36:50,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:50,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:51,691 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:53,692 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:53,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:53,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:54,330 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:55,693 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:36:55,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:55,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:56,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:36:56,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:36:56,530 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:36:58,689 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:58,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:36:59,687 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:36:59,694 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:00,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:00,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:01,695 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:03,696 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:04,165 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:37:04,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:04,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:05,028 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:07,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:07,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:07,697 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:09,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:09,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:09,698 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:10,370 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:11,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:37:11,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:37:11,530 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:37:11,699 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:12,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:12,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:14,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:14,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:15,700 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:15,718 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:17,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:17,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:17,701 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:19,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:19,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:19,702 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:20,159 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:37:21,062 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:22,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:22,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:23,703 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:24,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:24,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:25,704 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:26,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:26,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:37:26,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:37:26,530 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:37:27,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:27,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:27,704 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:29,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:29,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:31,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:31,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:31,706 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:31,754 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:33,707 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:34,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:34,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:35,708 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:36,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:36,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:37,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:37,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:37:37,132 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:37:37,132 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:37:37,133 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:37:37,709 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:37:39,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:39,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:39,709 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:41,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:37:41,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:37:41,530 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:37:41,710 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:42,452 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:42,732 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:37:42,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:42,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:43,711 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:44,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:44,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:47,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:47,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:47,712 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:47,802 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:49,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:49,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:49,712 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:50,161 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:37:51,714 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:52,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:52,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:53,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:54,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:54,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:55,715 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:56,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:37:56,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:37:56,530 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:37:56,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:56,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:57,716 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:37:58,503 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:37:59,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:59,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:37:59,716 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:01,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:01,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:03,718 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:03,832 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:04,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:04,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:05,719 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:06,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:06,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:07,719 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:09,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:09,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:09,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:11,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:38:11,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:38:11,531 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:38:11,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:11,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:11,721 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:13,722 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:14,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:14,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:14,513 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:15,723 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:16,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:16,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:19,724 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:19,854 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:19,931 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:38:19,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:19,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:20,162 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:38:21,725 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:22,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:22,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:23,725 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:24,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:24,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:25,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:26,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:38:26,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:38:26,531 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:38:27,727 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:27,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:27,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:29,727 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:29,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:29,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:30,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:31,728 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:32,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:32,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:34,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:34,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:35,729 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:35,891 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:37,729 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:38,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:38,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:39,730 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:40,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:40,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:41,241 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:41,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:38:41,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:38:41,531 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:38:42,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:42,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:43,731 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:43,938 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:38:43,940 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:38:43,940 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:38:43,940 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:38:44,732 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:38:45,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:45,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:45,732 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:46,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:47,733 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:48,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:48,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:49,733 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:38:50,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:50,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:50,163 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:38:52,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:52,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:52,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:54,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:54,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:56,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:38:56,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:38:56,531 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:38:57,735 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:38:57,904 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:38:57,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:38:57,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:00,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:00,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:02,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:02,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:02,736 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:03,738 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:05,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:05,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:07,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:07,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:07,739 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:08,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:09,739 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:10,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:10,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:11,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:39:11,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:39:11,531 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:39:11,741 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:12,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:12,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:13,725 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:14,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:14,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:15,742 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:17,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:17,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:17,742 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:19,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:19,652 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:19,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:19,743 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:20,165 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:39:22,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:22,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:23,744 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:24,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:24,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:24,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:25,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:26,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:39:26,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:39:26,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:39:27,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:27,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:27,746 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:29,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:29,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:29,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:31,747 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:32,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:32,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:33,747 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:35,051 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:35,748 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:36,220 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:39:36,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:36,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:39,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:39,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:39,749 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:40,382 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:41,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:41,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:41,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:39:41,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:39:41,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:39:41,749 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:43,750 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:44,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:44,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:45,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:46,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:46,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:47,751 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:49,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:49,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:49,752 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:50,167 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:39:51,034 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:51,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:51,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:51,753 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:54,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:54,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:55,754 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:56,377 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:39:56,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:39:56,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:39:56,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:39:56,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:56,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:57,754 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:39:59,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:59,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:39:59,755 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:01,713 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:01,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:01,740 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:40:01,741 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:40:01,742 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:40:01,742 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:40:01,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:01,756 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:40:03,757 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:03,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:03,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:05,757 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:06,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:06,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:07,052 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:07,758 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:08,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:08,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:11,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:40:11,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:40:11,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:40:11,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:11,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:11,759 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:12,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:13,760 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:15,374 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:40:15,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:15,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:15,760 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:17,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:17,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:17,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:19,762 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:20,169 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:40:20,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:20,234 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:21,762 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:22,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:22,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:23,081 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:23,763 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:25,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:25,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:26,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:40:26,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:40:26,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:40:27,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:27,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:27,764 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:28,434 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:29,765 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:30,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:30,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:31,766 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:32,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:32,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:33,777 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:35,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:35,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:35,766 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:37,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:37,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:37,768 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:39,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:39,768 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:40,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:40,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:41,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:40:41,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:40:41,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:40:42,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:42,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:43,770 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:44,468 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:45,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:45,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:45,770 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:47,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:47,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:47,771 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:49,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:49,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:49,806 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:50,170 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:40:51,772 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:52,959 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:40:52,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:52,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:53,773 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:55,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:40:55,773 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:40:55,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:55,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:56,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:40:56,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:40:56,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:40:57,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:57,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:40:59,774 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:00,486 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:00,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:00,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:01,775 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:02,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:02,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:03,775 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:04,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:04,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:05,825 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:07,777 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:08,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:08,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:08,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:41:08,528 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:41:08,528 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:41:08,529 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:41:08,777 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:41:09,778 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:10,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:10,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:11,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:11,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:41:11,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:41:11,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:41:11,778 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:12,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:12,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:15,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:15,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:15,780 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:16,507 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:17,693 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:17,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:17,781 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:19,782 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:20,173 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:41:20,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:20,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:21,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:22,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:22,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:23,783 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:25,783 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:25,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:25,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:26,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:41:26,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:41:26,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:41:27,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:27,784 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:27,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:27,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:31,382 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:41:31,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:31,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:31,786 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:32,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:33,787 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:34,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:34,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:35,787 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:36,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:36,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:37,865 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:39,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:39,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:39,789 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:41,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:41,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:41,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:41:41,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:41:41,533 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:41:41,789 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:43,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:43,790 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:44,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:44,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:46,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:46,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:47,791 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:48,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:49,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:49,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:49,791 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:50,174 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:41:51,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:51,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:51,792 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:53,873 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:54,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:54,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:55,793 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:56,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:41:56,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:41:56,532 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:41:56,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:56,800 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:57,794 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:59,209 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:41:59,795 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:41:59,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:41:59,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:01,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:01,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:03,801 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:03,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:03,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:04,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:05,801 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:06,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:06,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:07,802 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:09,885 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:10,525 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:42:10,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:10,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:11,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:42:11,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:42:11,533 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:42:11,803 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:12,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:12,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:13,804 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:15,210 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:15,237 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:42:15,238 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:42:15,238 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:42:15,239 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:42:15,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:15,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:15,804 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:42:15,805 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:17,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:17,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:19,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:19,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:19,806 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:20,176 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:42:20,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:21,807 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:22,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:22,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:23,807 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:24,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:24,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:25,888 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:26,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:42:26,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:42:26,533 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:42:27,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:27,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:27,809 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:29,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:29,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:29,810 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:31,224 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:31,810 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:32,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:32,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:34,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:34,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:35,812 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:36,557 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:37,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:37,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:37,812 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:39,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:39,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:39,813 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:41,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:42:41,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:42:41,534 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:42:41,757 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:42,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:42,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:43,814 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:44,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:44,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:45,815 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:47,218 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:47,816 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:47,837 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:42:47,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:47,865 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:50,177 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:42:50,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:50,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:51,817 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:52,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:52,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:52,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:53,817 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:55,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:55,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:55,818 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:42:56,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:42:56,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:42:56,533 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:42:57,749 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:42:57,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:57,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:42:59,819 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:00,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:00,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:01,821 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:02,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:02,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:03,219 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:03,821 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:05,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:05,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:07,822 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:07,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:07,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:08,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:09,823 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:10,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:10,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:11,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:43:11,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:43:11,534 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:43:11,823 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:12,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:12,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:13,730 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:15,825 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:15,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:15,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:17,825 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:17,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:17,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:19,214 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:19,826 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:20,179 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:43:20,956 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:20,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:21,914 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:43:21,916 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:43:21,917 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:43:21,917 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:43:22,827 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:43:22,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:22,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:23,827 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:24,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:25,828 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:26,461 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:43:26,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:26,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:26,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:43:26,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:43:26,534 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:43:27,829 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:29,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:29,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:29,895 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:31,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:31,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:31,830 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:33,832 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:34,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:34,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:35,226 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:35,832 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:36,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:36,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:39,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:39,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:39,833 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:40,559 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:41,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:41,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:43:41,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:43:41,534 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:43:41,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:41,834 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:43,834 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:44,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:44,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:45,896 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:46,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:46,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:47,836 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:49,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:49,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:49,836 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:50,180 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:43:51,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:51,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:51,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:51,837 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:54,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:54,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:55,838 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:56,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:43:56,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:43:56,535 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:43:56,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:43:56,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:56,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:57,839 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:43:59,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:59,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:43:59,840 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:01,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:01,693 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:01,901 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:03,841 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:05,028 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:44:05,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:05,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:05,842 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:07,226 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:07,842 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:07,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:07,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:10,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:10,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:11,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:44:11,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:44:11,535 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:44:11,844 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:12,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:12,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:12,566 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:13,844 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:14,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:14,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:15,845 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:16,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:16,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:17,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:19,847 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:19,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:19,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:20,182 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:44:21,848 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:21,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:21,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:23,221 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:23,848 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:24,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:24,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:26,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:44:26,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:44:26,535 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:44:26,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:26,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:27,849 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:28,562 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:28,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:44:28,590 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:44:28,590 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:44:28,591 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:44:28,850 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:44:28,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:29,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:29,850 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:31,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:31,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:31,852 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:33,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:33,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:33,902 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:36,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:36,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:36,853 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:38,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:38,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:38,854 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:39,225 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:40,854 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:41,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:44:41,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:44:41,536 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:44:42,280 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:44:42,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:42,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:44,557 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:44,856 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:45,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:45,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:46,857 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:47,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:47,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:48,858 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:49,900 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:50,183 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:44:50,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:50,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:52,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:52,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:52,859 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:54,859 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:55,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:55,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:55,229 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:44:56,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:44:56,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:44:56,535 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:44:56,860 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:44:57,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:44:57,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:00,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:00,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:00,568 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:00,861 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:02,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:02,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:02,862 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:04,863 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:05,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:05,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:05,908 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:07,206 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:07,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:08,864 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:10,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:10,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:10,864 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:11,252 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:11,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:45:11,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:45:11,535 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:45:12,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:12,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:12,865 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:15,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:15,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:16,595 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:16,867 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:17,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:17,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:18,867 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:20,185 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:45:20,868 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:20,874 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:45:20,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:20,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:21,914 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:23,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:23,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:24,870 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:25,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:25,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:26,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:45:26,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:45:26,535 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:45:26,871 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:27,239 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:28,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:28,793 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:28,873 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:30,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:30,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:32,579 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:32,873 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:33,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:33,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:34,874 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:35,273 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:45:35,275 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:45:35,275 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:45:35,275 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:45:35,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:35,771 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:35,874 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:45:35,875 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:36,875 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:37,917 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:38,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:38,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:40,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:40,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:40,877 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:41,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:45:41,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:45:41,535 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:45:42,877 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:43,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:43,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:43,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:44,878 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:45,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:45,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:48,596 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:48,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:48,865 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:48,879 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:50,187 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:45:50,880 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:50,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:50,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:52,881 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:53,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:53,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:54,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:45:55,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:55,767 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:56,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:45:56,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:45:56,535 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:45:56,882 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:58,882 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:45:59,077 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:45:59,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:59,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:45:59,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:00,883 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:01,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:01,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:03,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:03,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:04,884 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:04,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:06,885 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:06,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:06,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:08,886 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:09,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:09,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:10,245 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:11,046 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:11,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:11,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:46:11,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:46:11,536 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:46:12,887 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:13,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:13,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:14,887 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:15,587 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:15,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:15,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:16,888 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:17,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:17,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:20,188 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:46:20,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:20,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:20,890 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:20,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:22,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:22,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:22,890 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:24,891 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:25,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:25,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:26,297 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:26,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:46:26,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:46:26,536 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:46:27,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:27,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:28,893 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:30,893 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:30,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:30,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:31,637 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:32,894 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:32,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:32,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:36,470 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:46:36,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:36,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:36,895 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:36,978 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:38,896 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:39,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:39,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:40,897 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:41,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:41,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:41,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:46:41,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:46:41,536 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:46:42,327 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:42,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:46:42,354 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:46:42,354 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:46:42,354 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:46:42,898 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:46:43,898 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:44,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:44,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:44,898 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:46,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:46,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:46,899 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:47,681 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:48,900 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:49,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:49,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:50,191 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:46:51,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:51,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:52,901 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:53,030 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:54,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:54,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:54,901 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:56,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:46:56,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:46:56,536 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:46:56,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:56,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:56,902 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:46:58,375 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:46:59,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:46:59,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:00,903 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:01,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:01,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:02,904 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:03,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:04,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:04,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:04,905 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:06,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:06,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:08,905 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:09,060 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:09,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:09,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:10,906 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:11,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:11,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:11,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:47:11,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:47:11,536 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:47:14,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:14,907 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:14,987 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:47:15,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:15,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:16,908 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:17,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:17,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:18,908 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:19,760 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:20,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:20,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:20,192 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:47:20,909 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:22,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:22,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:24,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:24,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:24,910 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:25,105 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:26,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:47:26,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:47:26,536 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:47:26,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:26,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:26,911 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:28,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:28,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:28,912 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:30,451 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:31,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:31,625 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:32,913 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:33,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:33,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:34,914 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:35,798 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:36,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:36,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:38,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:38,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:38,915 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:40,916 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:41,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:41,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:47:41,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:47:41,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:47:41,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:41,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:42,916 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:43,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:43,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:46,504 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:46,917 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:46,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:46,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:48,918 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:48,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:48,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:49,205 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:47:49,206 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:47:49,207 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:47:49,207 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:47:49,918 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:47:49,919 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:50,193 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:47:50,919 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:51,856 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:52,426 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:47:52,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:52,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:54,920 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:55,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:55,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:56,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:47:56,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:47:56,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:47:56,921 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:47:57,204 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:47:57,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:57,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:47:58,922 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:00,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:00,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:02,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:02,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:02,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:02,923 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:04,924 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:05,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:05,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:06,925 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:07,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:07,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:07,885 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:10,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:10,689 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:10,926 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:11,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:48:11,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:48:11,537 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:48:12,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:12,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:12,926 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:13,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:14,928 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:15,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:15,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:17,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:17,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:18,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:18,929 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:20,195 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:48:20,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:20,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:20,930 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:22,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:22,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:22,930 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:23,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:24,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:24,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:26,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:48:26,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:48:26,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:48:26,931 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:27,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:27,419 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:28,932 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:29,282 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:30,933 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:31,059 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:48:31,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:31,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:33,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:33,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:34,635 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:34,934 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:35,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:35,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:36,935 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:37,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:37,956 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:38,935 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:39,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:39,982 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:39,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:41,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:48:41,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:48:41,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:48:42,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:42,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:42,936 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:44,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:44,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:44,937 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:45,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:46,938 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:47,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:47,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:49,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:49,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:50,197 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:48:50,677 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:50,939 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:52,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:52,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:52,940 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:54,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:54,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:54,941 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:48:56,101 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:48:56,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:48:56,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:48:56,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:48:57,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:57,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:59,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:48:59,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:01,770 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:02,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:02,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:04,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:04,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:06,770 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:07,018 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:49:07,020 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:49:07,020 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:49:07,020 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:49:07,483 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:49:07,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:07,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:07,945 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:49:08,946 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:10,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:10,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:10,947 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:11,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:49:11,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:49:11,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:49:12,308 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:12,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:12,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:14,948 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:15,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:15,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:16,948 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:17,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:17,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:17,637 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:18,950 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:20,198 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:49:20,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:20,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:22,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:22,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:22,951 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:22,967 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:24,952 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:25,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:25,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:26,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:49:26,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:49:26,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:49:26,952 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:27,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:27,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:28,304 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:30,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:30,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:30,954 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:32,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:32,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:32,955 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:33,644 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:34,955 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:35,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:35,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:37,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:37,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:38,956 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:38,980 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:40,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:40,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:40,958 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:41,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:49:41,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:49:41,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:49:42,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:42,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:42,958 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:44,310 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:46,443 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:49:46,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:46,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:46,960 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:48,960 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:49,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:49,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:49,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:50,200 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:49:50,961 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:51,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:51,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:54,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:54,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:54,963 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:54,974 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:49:56,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:56,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:56,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:49:56,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:49:56,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:49:56,963 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:49:58,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:58,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:49:58,964 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:00,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:01,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:01,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:02,965 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:03,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:03,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:04,966 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:05,645 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:06,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:06,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:06,967 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:08,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:08,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:10,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:10,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:10,968 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:10,969 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:11,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:50:11,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:50:11,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:50:12,969 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:13,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:13,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:13,662 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:50:13,664 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:50:13,665 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:50:13,665 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:50:13,969 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:50:14,970 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:15,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:15,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:16,305 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:18,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:18,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:18,972 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:20,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:20,130 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:20,201 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:50:20,972 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:21,640 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:22,973 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:23,571 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:50:23,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:23,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:26,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:26,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:26,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:50:26,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:50:26,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:50:26,683 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:26,975 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:28,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:28,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:28,975 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:30,976 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:31,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:31,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:32,308 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:33,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:33,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:34,978 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:36,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:36,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:36,978 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:37,638 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:38,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:38,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:38,979 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:41,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:41,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:41,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:50:41,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:50:41,539 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:50:42,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:42,980 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:43,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:43,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:44,981 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:46,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:46,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:46,981 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:48,287 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:48,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:48,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:50,202 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:50:50,982 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:51,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:51,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:52,983 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:53,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:53,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:53,614 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:54,984 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:50:56,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:56,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:56,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:50:56,538 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:50:56,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:50:58,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:58,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:50:58,676 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:50:58,986 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:00,986 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:01,896 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:51:01,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:01,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:02,987 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:04,276 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:04,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:04,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:06,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:06,771 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:06,989 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:08,990 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:09,590 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:09,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:09,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:10,990 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:11,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:51:11,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:51:11,539 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:51:11,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:11,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:14,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:14,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:14,919 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:14,992 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:16,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:16,935 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:16,993 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:18,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:19,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:19,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:20,204 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:51:20,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:20,271 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:51:20,272 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:51:20,272 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:51:20,273 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:51:20,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:51:21,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:21,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:22,996 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:24,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:24,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:24,997 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:25,575 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:26,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:51:26,539 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:51:26,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:51:26,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:26,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:26,998 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:29,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:29,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:30,919 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:31,000 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:32,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:32,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:33,001 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:34,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:34,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:35,001 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:36,254 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:36,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:36,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:39,003 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:40,451 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:51:40,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:40,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:41,005 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:41,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:41,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:51:41,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:51:41,539 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:51:43,006 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:43,467 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:43,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:45,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:45,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:46,899 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:47,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:48,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:48,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:49,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:50,205 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:51:50,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:50,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:51,009 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:52,207 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:53,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:53,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:55,010 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:55,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:55,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:56,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:51:56,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:51:56,540 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:51:57,011 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:51:57,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:51:58,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:58,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:51:59,012 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:00,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:00,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:02,849 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:03,014 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:03,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:03,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:05,015 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:05,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:05,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:07,016 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:08,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:08,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:08,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:10,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:10,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:11,017 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:11,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:52:11,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:52:11,539 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:52:13,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:13,509 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:13,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:13,956 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:15,019 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:15,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:15,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:17,019 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:18,838 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:19,458 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:52:19,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:19,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:20,207 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:52:21,021 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:22,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:22,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:23,022 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:24,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:24,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:24,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:25,022 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:26,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:52:26,539 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:52:26,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:52:26,843 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:52:26,845 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:52:26,845 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:52:26,846 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:52:27,023 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:52:27,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:27,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:29,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:29,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:29,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:29,481 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:31,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:32,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:32,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:33,025 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:34,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:34,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:34,810 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:37,026 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:37,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:37,204 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:39,028 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:39,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:39,235 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:40,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:41,028 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:41,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:52:41,540 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:52:41,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:52:42,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:42,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:44,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:44,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:45,030 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:45,454 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:47,030 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:47,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:47,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:49,031 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:49,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:49,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:50,208 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:52:50,786 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:52,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:52,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:53,032 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:54,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:54,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:55,033 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:56,101 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:52:56,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:52:56,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:52:56,540 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:52:57,034 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:52:58,017 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:52:58,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:52:58,046 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:00,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:00,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:01,035 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:01,422 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:02,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:03,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:03,036 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:05,037 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:05,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:05,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:06,753 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:07,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:07,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:09,037 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:10,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:10,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:11,038 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:11,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:53:11,540 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:53:11,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:53:12,087 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:12,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:12,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:13,039 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:15,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:15,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:17,040 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:17,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:17,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:17,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:19,041 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:20,210 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:53:20,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:20,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:21,041 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:22,746 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:23,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:23,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:25,043 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:25,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:25,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:26,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:53:26,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:53:26,540 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:53:27,043 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:27,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:27,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:28,072 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:29,044 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:30,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:30,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:33,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:33,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:33,045 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:33,388 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:33,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:53:33,416 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:53:33,416 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:53:33,417 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:53:34,045 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:53:35,046 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:36,521 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:53:36,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:36,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:37,047 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:38,724 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:39,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:39,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:41,048 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:41,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:41,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:41,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:53:41,540 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:53:41,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:53:43,048 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:44,067 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:44,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:44,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:45,050 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:46,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:46,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:49,051 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:49,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:49,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:49,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:50,212 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:53:51,052 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:51,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:51,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:53,052 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:53,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:53,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:54,723 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:53:56,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:56,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:56,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:53:56,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:53:56,540 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:53:57,054 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:53:58,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:58,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:53:59,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:00,059 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:01,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:01,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:01,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:03,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:03,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:05,056 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:05,379 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:06,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:06,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:07,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:08,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:08,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:09,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:10,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:11,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:11,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:11,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:54:11,540 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:54:11,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:54:13,060 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:15,037 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:54:15,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:15,060 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:15,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:16,039 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:17,061 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:18,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:18,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:20,214 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:54:20,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:20,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:21,063 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:21,216 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:23,063 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:23,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:23,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:25,064 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:25,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:25,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:26,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:26,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:54:26,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:54:26,541 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:54:28,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:28,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:29,066 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:30,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:30,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:31,067 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:32,013 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:33,067 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:33,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:33,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:35,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:35,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:37,068 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:37,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:38,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:38,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:39,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:40,034 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:54:40,035 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:54:40,036 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:54:40,036 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:54:40,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:54:40,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:40,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:41,070 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:41,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:54:41,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:54:41,541 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:54:42,679 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:43,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:43,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:45,072 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:45,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:45,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:47,072 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:47,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:48,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:48,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:49,073 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:50,215 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:54:50,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:50,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:53,075 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:53,340 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:54,472 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:54:54,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:54,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:55,076 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:56,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:54:56,541 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:54:56,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:54:57,077 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:54:57,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:57,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:58,666 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:54:59,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:54:59,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:01,078 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:02,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:02,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:03,079 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:03,997 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:04,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:04,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:05,080 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:06,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:06,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:09,081 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:09,309 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:09,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:09,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:11,082 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:11,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:11,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:11,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:55:11,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:55:11,541 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:55:13,082 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:13,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:13,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:14,655 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:16,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:16,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:17,084 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:18,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:18,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:19,085 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:19,996 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:20,216 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:55:20,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:20,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:21,085 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:23,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:23,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:25,087 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:25,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:25,324 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:25,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:26,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:55:26,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:55:26,542 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:55:27,088 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:28,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:28,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:29,089 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:30,661 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:32,024 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:55:32,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:32,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:33,090 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:34,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:34,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:35,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:35,989 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:37,092 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:37,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:37,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:39,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:39,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:41,093 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:41,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:41,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:41,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:41,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:55:41,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:55:41,542 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:55:43,094 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:44,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:44,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:45,094 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:46,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:46,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:46,658 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:46,683 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:55:46,685 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:55:46,685 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:55:46,686 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:55:47,095 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:55:49,096 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:49,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:49,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:50,218 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:55:51,097 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:51,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:51,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:51,981 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:53,098 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:54,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:54,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:56,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:56,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:56,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:55:56,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:55:56,542 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:55:57,099 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:57,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:55:59,100 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:55:59,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:55:59,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:01,101 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:01,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:01,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:02,638 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:03,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:03,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:05,102 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:06,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:06,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:07,103 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:07,963 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:09,104 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:09,433 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:56:10,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:10,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:11,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:56:11,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:56:11,542 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:56:12,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:12,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:13,106 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:13,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:15,107 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:15,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:15,467 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:17,108 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:17,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:17,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:18,623 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:20,220 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:56:20,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:20,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:21,109 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:22,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:22,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:23,110 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:23,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:24,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:24,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:25,110 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:26,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:56:26,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:56:26,542 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:56:27,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:27,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:29,112 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:29,273 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:29,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:29,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:31,112 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:31,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:31,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:33,113 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:34,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:34,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:34,591 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:36,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:36,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:37,115 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:38,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:38,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:39,116 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:39,917 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:40,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:40,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:41,116 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:41,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:56:41,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:56:41,542 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:56:42,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:42,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:45,118 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:45,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:46,452 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:56:46,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:46,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:47,119 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:49,120 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:49,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:49,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:50,222 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:56:50,571 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:51,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:51,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:53,121 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:53,262 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:56:53,263 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:56:53,263 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:56:53,264 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:56:53,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:53,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:54,122 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:56:55,122 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:55,896 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:56:56,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:56,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:56,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:56:56,542 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:56:56,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:56:57,123 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:56:58,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:58,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:56:59,123 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:01,219 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:01,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:01,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:03,125 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:03,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:03,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:05,126 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:06,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:06,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:06,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:07,126 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:08,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:08,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:11,128 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:11,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:11,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:11,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:57:11,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:57:11,543 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:57:11,755 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:13,129 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:13,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:13,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:15,129 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:16,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:16,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:17,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:18,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:18,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:19,130 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:20,223 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:57:21,131 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:21,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:21,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:22,224 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:23,132 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:24,058 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:57:24,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:24,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:26,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:26,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:26,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:57:26,542 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:57:26,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:57:27,133 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:27,721 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:28,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:28,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:29,134 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:31,135 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:31,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:31,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:33,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:33,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:33,204 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:35,137 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:36,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:36,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:37,137 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:38,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:38,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:38,447 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:39,138 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:41,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:41,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:41,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:57:41,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:57:41,543 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:57:43,140 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:43,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:43,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:43,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:45,141 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:45,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:46,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:47,141 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:48,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:48,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:49,090 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:50,225 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:57:51,143 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:51,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:51,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:53,144 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:53,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:53,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:54,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:55,145 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:55,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:55,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:56,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:57:56,543 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:57:56,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:57:58,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:58,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:57:59,146 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:57:59,729 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:57:59,755 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:57:59,757 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:57:59,757 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:57:59,757 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:58:00,147 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:58:01,147 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:02,037 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:58:02,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:02,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:03,148 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:04,090 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:04,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:05,046 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:06,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:06,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:07,149 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:08,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:08,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:09,149 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:10,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:11,151 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:11,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:58:11,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:58:11,543 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:58:11,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:11,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:13,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:13,866 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:15,152 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:15,686 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:16,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:16,879 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:17,152 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:18,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:18,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:19,153 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:20,227 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:58:21,004 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:21,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:21,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:23,154 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:23,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:23,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:25,155 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:26,332 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:26,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:58:26,543 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:58:26,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:58:27,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:27,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:27,155 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:29,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:29,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:31,157 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:31,651 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:31,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:32,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:33,157 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:34,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:34,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:35,158 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:36,972 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:37,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:37,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:39,159 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:40,616 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:58:40,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:40,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:41,160 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:41,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:58:41,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:58:41,543 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:58:42,293 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:42,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:42,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:43,160 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:45,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:45,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:47,162 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:47,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:47,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:47,617 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:49,162 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:50,229 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:58:50,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:50,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:51,163 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:52,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:52,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:52,932 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:55,164 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:55,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:55,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:56,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:58:56,543 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:58:56,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:58:57,165 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:58:57,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:57,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:58,763 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:58:59,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:58:59,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:01,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:01,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:03,763 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:04,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:04,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:06,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:06,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:08,764 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:09,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:09,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:11,170 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:11,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:11,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:11,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:59:11,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:59:11,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:59:13,170 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:14,076 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:14,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:14,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:15,171 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:17,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 00:59:17,415 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 00:59:17,416 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 00:59:17,416 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 00:59:17,887 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:59:17,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:17,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:18,173 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 00:59:19,173 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:19,418 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:20,230 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:59:20,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:20,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:22,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:22,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:25,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:25,232 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:25,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:26,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:59:26,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:59:26,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:59:27,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:27,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:29,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:29,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:30,926 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:31,177 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:31,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:31,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:34,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:34,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:35,178 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:36,507 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:36,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:36,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:39,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:39,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:41,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:41,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:41,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:41,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:59:41,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:59:41,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:59:42,180 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:44,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:44,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:46,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:46,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:46,748 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:48,181 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:49,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:49,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:50,232 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 00:59:51,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:51,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:52,327 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:54,183 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 00:59:54,784 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 00:59:54,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:54,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:56,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 00:59:56,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 00:59:56,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 00:59:57,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:57,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:58,000 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 00:59:59,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 00:59:59,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:00,185 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:02,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:02,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:03,575 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:04,186 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:04,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:04,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:07,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:07,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:09,221 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:09,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:09,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:10,188 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:11,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:00:11,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:00:11,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:00:12,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:12,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:14,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:14,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:14,808 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:16,190 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:16,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:17,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:19,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:19,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:20,233 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:00:20,235 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:21,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:21,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:22,192 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:23,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:23,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:26,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:26,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:00:26,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:00:26,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:00:26,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:26,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:28,194 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:28,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:28,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:31,693 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:32,185 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:00:32,195 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:32,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:32,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:34,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:34,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:36,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:36,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:37,293 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:38,197 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:39,940 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:39,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:41,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:00:41,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:00:41,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:00:41,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:41,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:42,857 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:44,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:44,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:44,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:46,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:46,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:48,456 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:49,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:49,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:50,200 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:50,235 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:00:51,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:51,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:54,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:00:54,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:54,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:56,202 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:00:56,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:00:56,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:00:56,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:00:56,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:56,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:59,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:59,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:00:59,714 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:00,204 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:01,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:01,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:04,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:04,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:05,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:06,206 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:06,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:06,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:09,649 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:01:09,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:09,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:10,944 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:11,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:01:11,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:01:11,544 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:01:12,207 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:12,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:12,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:14,414 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:14,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:16,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:17,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:17,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:18,209 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:19,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:19,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:20,236 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:01:21,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:21,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:22,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:23,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:24,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:24,211 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:26,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:01:26,545 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:01:26,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:01:26,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:26,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:27,786 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:28,212 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:28,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:28,961 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:31,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:31,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:33,440 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:33,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:33,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:34,214 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:36,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:36,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:38,757 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:38,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:39,035 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:40,215 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:41,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:01:41,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:01:41,545 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:01:41,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:41,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:43,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:43,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:44,641 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:46,217 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:46,788 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:01:46,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:46,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:49,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:49,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:50,238 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:01:50,240 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:51,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:51,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:52,219 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:54,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:54,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:55,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:01:56,220 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:01:56,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:56,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:56,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:01:56,545 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:01:56,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:01:59,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:01:59,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:01,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:01,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:01,495 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:02,221 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:04,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:04,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:06,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:06,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:07,117 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:08,223 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:08,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:08,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:11,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:11,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:11,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:02:11,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:02:11,546 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:02:12,785 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:13,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:13,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:14,225 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:15,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:16,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:18,371 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:18,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:18,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:20,227 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:20,239 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:02:20,767 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:20,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:23,949 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:23,996 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:02:24,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:24,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:24,228 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:26,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:02:26,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:02:26,545 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:02:26,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:26,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:28,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:28,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:29,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:30,230 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:31,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:31,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:33,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:33,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:35,250 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:36,231 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:36,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:36,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:38,712 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:38,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:40,769 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:41,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:02:41,545 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:02:41,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:02:41,595 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:41,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:42,233 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:43,625 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:43,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:45,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:45,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:46,385 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:48,235 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:48,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:48,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:50,241 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:02:50,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:50,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:51,940 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:52,236 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:02:53,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:53,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:55,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:55,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:56,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:02:56,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:02:56,546 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:02:57,695 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:02:57,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:57,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:02:58,238 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:01,316 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:03:01,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:01,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:03,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:04,239 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:04,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:04,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:06,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:06,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:08,764 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:09,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:09,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:10,241 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:11,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:11,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:11,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:03:11,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:03:11,546 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:03:13,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:13,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:14,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:15,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:15,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:16,243 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:18,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:18,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:19,919 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:20,243 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:03:20,246 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:20,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:20,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:23,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:23,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:25,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:25,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:25,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:26,247 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:26,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:03:26,546 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:03:26,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:03:28,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:28,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:30,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:30,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:31,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:32,249 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:33,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:33,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:35,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:35,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:36,724 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:38,251 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:38,485 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:03:38,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:38,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:41,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:41,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:41,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:03:41,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:03:41,546 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:03:41,742 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:43,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:43,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:44,252 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:46,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:46,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:46,742 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:48,253 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:48,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:48,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:50,245 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:03:51,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:51,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:52,246 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:53,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:53,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:54,255 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:03:56,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:56,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:56,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:03:56,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:03:56,546 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:03:57,714 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:03:58,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:03:58,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:00,256 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:01,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:01,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:03,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:03,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:03,285 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:06,258 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:06,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:06,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:08,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:08,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:08,950 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:11,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:11,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:11,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:04:11,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:04:11,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:04:12,260 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:13,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:13,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:14,692 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:16,261 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:16,646 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:04:16,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:16,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:19,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:19,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:20,109 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:20,249 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:04:21,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:21,467 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:22,265 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:24,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:24,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:25,699 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:26,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:26,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:26,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:04:26,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:04:26,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:04:28,267 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:29,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:29,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:30,734 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:31,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:31,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:34,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:34,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:34,268 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:36,012 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:36,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:36,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:38,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:38,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:40,270 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:40,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:40,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:41,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:41,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:04:41,546 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:04:41,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:04:43,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:43,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:45,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:45,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:46,271 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:47,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:48,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:48,625 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:50,250 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:04:50,273 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:50,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:50,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:52,252 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:53,953 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:04:53,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:53,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:56,275 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:04:56,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:04:56,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:04:56,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:04:56,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:56,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:57,687 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:04:58,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:04:58,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:01,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:01,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:02,277 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:03,058 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:03,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:03,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:06,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:06,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:08,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:08,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:08,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:08,608 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:11,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:11,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:11,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:05:11,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:05:11,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:05:12,279 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:13,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:13,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:13,698 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:16,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:16,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:18,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:18,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:18,281 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:18,782 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:20,252 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:05:21,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:21,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:23,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:23,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:24,282 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:24,423 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:26,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:26,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:26,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:05:26,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:05:26,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:05:28,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:28,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:30,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:30,284 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:31,541 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:05:31,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:31,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:34,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:34,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:35,640 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:36,286 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:36,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:36,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:39,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:39,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:40,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:41,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:41,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:41,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:41,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:05:41,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:05:41,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:05:44,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:44,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:46,289 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:46,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:46,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:46,813 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:49,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:49,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:50,254 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:05:51,235 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:51,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:52,291 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:52,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:54,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:54,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:56,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:56,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:56,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:05:56,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:05:56,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:05:57,699 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:05:58,292 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:05:59,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:05:59,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:01,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:01,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:03,650 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:04,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:04,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:04,294 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:06,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:06,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:08,296 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:09,250 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:09,394 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:06:09,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:09,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:11,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:06:11,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:06:11,548 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:06:12,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:12,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:14,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:14,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:14,297 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:14,830 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:17,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:17,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:19,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:19,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:20,256 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:06:20,258 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:20,299 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:22,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:22,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:24,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:24,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:26,030 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:26,300 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:26,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:06:26,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:06:26,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:06:26,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:26,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:29,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:29,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:31,697 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:32,046 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:32,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:32,302 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:34,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:34,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:36,303 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:36,879 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:36,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:37,243 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:38,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:38,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:41,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:06:41,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:06:41,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:06:41,693 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:41,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:42,305 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:42,686 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:43,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:43,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:47,095 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:06:47,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:47,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:48,307 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:48,377 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:49,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:49,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:50,258 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:06:51,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:51,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:53,961 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:54,309 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:06:54,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:54,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:56,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:06:56,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:06:56,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:06:56,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:56,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:59,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:06:59,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:06:59,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:00,311 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:01,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:01,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:04,312 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:04,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:04,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:05,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:06,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:06,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:09,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:09,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:10,314 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:10,772 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:11,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:11,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:11,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:07:11,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:07:11,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:07:14,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:14,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:16,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:16,242 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:16,315 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:16,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:19,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:19,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:20,260 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:07:21,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:21,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:22,261 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:22,317 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:24,609 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:07:24,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:24,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:26,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:07:26,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:07:26,548 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:07:27,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:27,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:27,724 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:28,319 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:29,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:29,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:32,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:32,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:32,320 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:33,087 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:34,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:34,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:37,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:37,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:38,322 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:38,690 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:39,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:39,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:41,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:07:41,547 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:07:41,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:07:42,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:42,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:44,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:44,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:44,265 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:44,323 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:47,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:47,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:49,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:49,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:49,893 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:50,261 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:07:50,325 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:52,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:52,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:54,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:54,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:55,479 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:07:56,326 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:07:56,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:07:56,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:07:56,548 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:07:56,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:56,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:58,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:07:58,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:00,328 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:01,039 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:02,150 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:08:02,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:02,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:05,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:05,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:06,329 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:06,610 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:07,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:07,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:10,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:10,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:11,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:08:11,548 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:08:11,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:08:11,701 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:12,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:12,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:12,331 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:14,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:14,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:16,715 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:16,851 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:16,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:18,333 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:19,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:19,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:20,263 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:08:21,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:21,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:22,299 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:22,334 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:24,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:24,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:26,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:26,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:26,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:08:26,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:08:26,548 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:08:27,879 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:28,336 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:29,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:29,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:31,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:31,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:33,617 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:34,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:34,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:34,337 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:36,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:36,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:36,697 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:08:36,699 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:08:36,699 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:08:36,700 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:08:37,338 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:08:38,339 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:38,702 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:39,148 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:08:39,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:39,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:41,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:41,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:43,703 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:44,305 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:08:44,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:08:44,307 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:08:44,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:44,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:46,342 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:08:47,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:47,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:49,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:49,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:50,425 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:08:50,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:52,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:52,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:55,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:55,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:55,428 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:08:57,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:57,309 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:59,516 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:08:59,516 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:08:59,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:08:59,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:08:59,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:00,721 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:01,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:01,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:04,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:04,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:05,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:06,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:06,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:08,346 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:09,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:09,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:11,087 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:11,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:11,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:12,347 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:14,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:14,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:14,348 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:14,387 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:09:14,388 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:09:14,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:09:16,349 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:16,384 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:17,752 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:09:17,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:17,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:18,349 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:20,267 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:09:20,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:20,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:21,686 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:22,351 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:22,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:22,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:24,351 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:25,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:25,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:26,352 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:26,987 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:27,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:27,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:29,387 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:09:29,388 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:09:29,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:09:30,353 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:30,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:30,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:32,301 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:32,353 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:32,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:32,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:34,354 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:35,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:35,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:37,628 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:38,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:38,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:38,355 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:40,356 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:41,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:41,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:42,357 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:42,949 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:43,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:43,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:44,388 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:09:44,388 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:09:44,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:09:45,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:45,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:46,358 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:48,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:48,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:48,257 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:48,360 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:50,268 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:09:50,360 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:51,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:51,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:53,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:53,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:53,565 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:54,362 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:56,362 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:56,815 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:09:56,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:56,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:58,363 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:09:58,882 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:09:59,387 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:09:59,388 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:09:59,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:09:59,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:09:59,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:01,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:01,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:02,364 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:04,199 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:04,365 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:04,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:04,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:06,366 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:06,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:06,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:09,523 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:09,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:09,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:10,367 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:11,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:11,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:12,368 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:13,657 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:13,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:13,869 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:10:13,870 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:10:13,870 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:10:13,871 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:10:14,369 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:10:14,370 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:14,388 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:10:14,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:10:14,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:10:14,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:16,370 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:16,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:16,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:18,371 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:18,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:18,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:20,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:20,269 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:10:20,372 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:21,625 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:21,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:22,373 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:23,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:23,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:25,482 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:26,375 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:26,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:26,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:28,376 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:28,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:28,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:29,388 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:10:29,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:10:29,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:10:30,376 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:30,806 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:31,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:31,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported