diff --git "a/wandb/run-20240327_190418-lwtbcr8s/logs/debug-internal.log" "b/wandb/run-20240327_190418-lwtbcr8s/logs/debug-internal.log" --- "a/wandb/run-20240327_190418-lwtbcr8s/logs/debug-internal.log" +++ "b/wandb/run-20240327_190418-lwtbcr8s/logs/debug-internal.log" @@ -32600,3 +32600,5390 @@ 2024-03-28 01:10:30,806 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report 2024-03-28 01:10:31,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported 2024-03-28 01:10:31,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:34,378 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:35,186 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:10:35,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:35,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:36,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:36,379 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:38,380 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:38,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:38,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:40,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:40,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:41,433 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:42,381 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:43,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:43,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:44,382 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:44,388 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:10:44,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:10:44,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:10:45,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:45,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:46,383 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:46,754 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:48,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:48,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:50,272 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:10:50,385 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:50,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:50,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:52,071 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:52,385 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:53,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:53,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:54,386 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:55,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:55,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:57,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:10:57,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:57,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:10:59,387 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:10:59,388 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:10:59,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:10:59,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:11:00,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:00,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:01,388 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:02,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:02,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:02,715 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:03,389 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:05,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:05,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:07,390 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:07,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:07,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:08,027 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:09,390 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:09,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:09,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:11,391 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:12,659 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:11:12,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:12,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:13,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:14,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:11:14,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:11:14,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:11:15,392 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:15,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:15,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:17,393 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:17,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:17,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:18,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:19,394 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:20,273 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:11:20,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:11:20,346 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:11:20,346 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:11:20,346 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:11:20,394 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:11:20,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:20,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:22,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:22,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:22,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:23,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:23,980 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:24,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:24,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:25,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:27,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:27,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:27,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:29,307 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:29,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:11:29,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:11:29,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:29,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:29,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:11:31,397 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:31,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:31,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:33,398 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:34,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:34,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:34,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:35,398 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:36,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:36,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:39,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:39,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:39,399 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:39,965 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:41,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:41,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:41,400 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:43,401 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:44,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:44,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:44,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:11:44,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:11:44,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:11:45,609 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:46,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:46,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:47,402 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:49,402 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:49,922 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:11:49,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:49,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:50,275 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:11:51,276 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:51,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:52,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:52,935 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:53,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:54,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:54,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:56,910 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:11:57,405 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:57,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:57,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:11:59,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:11:59,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:11:59,405 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:11:59,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:11:59,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:00,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:01,406 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:02,238 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:02,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:02,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:04,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:04,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:05,407 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:07,408 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:07,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:07,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:07,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:09,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:09,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:09,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:11,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:11,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:12,874 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:13,415 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:14,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:12:14,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:12:14,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:12:14,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:14,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:15,416 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:16,902 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:16,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:17,417 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:18,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:19,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:19,956 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:20,276 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:12:21,418 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:21,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:21,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:23,419 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:23,509 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:24,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:24,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:25,420 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:26,849 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:12:26,850 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:12:26,851 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:12:26,851 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:12:27,265 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:12:27,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:27,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:27,420 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:12:28,421 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:28,852 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:29,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:12:29,389 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:12:29,421 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:29,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:12:30,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:30,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:31,422 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:32,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:32,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:33,423 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:35,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:35,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:37,424 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:37,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:37,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:39,425 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:39,477 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:40,595 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:40,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:41,425 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:42,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:42,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:44,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:12:44,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:12:44,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:12:44,599 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:45,427 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:45,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:45,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:47,427 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:47,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:47,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:49,429 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:50,116 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:50,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:12:50,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:50,732 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:52,751 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:52,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:53,430 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:55,431 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:55,441 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:12:55,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:55,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:57,431 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:12:57,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:57,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:12:59,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:12:59,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:12:59,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:13:00,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:00,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:00,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:01,433 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:02,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:02,755 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:03,434 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:05,435 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:06,077 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:06,102 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:13:06,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:06,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:09,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:09,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:09,436 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:11,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:11,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:11,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:11,437 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:13,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:13,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:13,437 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:14,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:13:14,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:13:14,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:13:16,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:16,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:16,714 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:17,439 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:18,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:18,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:19,439 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:20,279 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:13:21,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:21,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:21,441 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:22,280 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:23,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:23,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:25,442 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:26,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:26,204 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:27,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:27,443 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:28,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:28,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:29,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:13:29,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:13:29,443 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:29,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:13:31,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:31,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:32,679 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:33,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:33,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:33,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:13:33,363 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:13:33,363 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:13:33,364 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:13:33,445 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:13:33,445 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:34,445 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:35,445 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:36,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:36,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:37,446 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:37,996 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:38,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:38,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:41,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:41,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:41,448 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:43,318 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:43,448 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:44,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:13:44,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:13:44,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:13:44,687 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:13:44,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:44,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:45,449 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:46,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:46,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:48,630 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:49,450 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:49,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:49,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:50,280 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:13:51,451 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:51,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:51,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:53,452 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:53,945 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:54,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:54,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:56,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:56,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:57,453 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:59,269 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:13:59,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:13:59,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:13:59,454 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:13:59,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:13:59,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:13:59,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:01,454 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:01,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:01,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:04,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:04,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:04,594 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:05,456 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:06,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:06,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:07,457 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:09,457 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:09,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:09,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:09,913 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:11,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:11,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:13,460 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:14,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:14:14,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:14:14,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:14:14,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:14,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:15,461 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:15,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:16,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:16,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:17,461 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:19,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:19,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:20,282 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:14:21,212 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:21,463 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:23,410 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:14:23,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:23,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:23,464 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:25,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:25,465 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:25,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:26,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:28,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:28,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:29,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:14:29,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:14:29,466 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:29,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:14:30,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:30,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:31,466 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:31,603 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:33,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:33,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:33,467 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:35,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:35,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:37,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:37,468 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:38,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:38,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:39,469 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:39,870 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:14:39,871 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:14:39,872 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:14:39,872 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:14:40,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:40,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:40,470 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:14:41,470 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:42,470 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:42,506 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:43,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:43,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:43,471 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:44,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:14:44,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:14:44,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:14:45,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:45,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:47,472 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:47,834 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:48,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:48,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:49,473 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:50,284 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:14:50,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:50,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:51,474 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:53,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:53,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:53,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:55,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:55,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:55,475 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:57,476 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:58,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:58,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:14:58,471 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:14:59,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:14:59,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:14:59,477 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:14:59,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:15:01,941 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:15:01,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:01,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:03,478 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:03,785 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:05,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:05,100 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:05,479 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:07,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:07,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:07,480 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:09,102 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:10,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:10,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:11,482 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:12,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:12,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:13,482 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:14,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:15:14,391 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:15:14,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:14,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:15:15,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:15,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:15,483 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:17,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:17,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:19,485 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:19,751 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:20,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:20,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:20,285 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:15:21,485 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:22,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:22,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:23,486 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:25,060 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:25,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:25,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:27,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:27,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:27,487 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:29,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:15:29,390 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:15:29,489 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:29,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:15:30,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:30,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:30,369 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:31,489 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:32,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:32,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:34,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:34,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:35,490 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:35,680 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:36,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:36,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:37,491 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:39,491 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:40,473 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:15:40,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:40,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:41,006 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:43,493 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:43,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:43,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:44,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:15:44,391 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:15:44,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:15:45,493 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:45,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:45,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:46,326 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:46,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:15:46,354 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:15:46,355 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:15:46,355 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:15:46,494 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:15:47,494 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:48,494 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:48,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:48,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:50,287 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:15:50,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:50,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:51,495 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:51,647 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:53,496 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:53,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:53,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:55,497 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:55,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:55,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:56,974 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:15:58,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:58,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:15:59,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:15:59,391 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:15:59,498 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:15:59,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:16:00,595 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:00,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:01,499 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:02,302 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:03,500 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:03,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:03,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:05,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:05,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:07,501 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:07,625 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:08,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:08,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:09,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:10,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:10,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:11,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:12,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:13,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:13,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:14,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:16:14,391 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:16:14,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:16:15,505 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:15,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:15,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:17,506 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:18,278 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:19,057 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:16:19,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:19,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:19,506 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:20,289 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:16:21,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:21,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:23,508 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:23,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:23,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:24,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:25,509 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:26,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:27,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:27,510 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:28,921 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:29,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:29,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:29,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:16:29,391 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:16:29,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:16:31,511 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:32,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:32,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:33,511 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:34,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:34,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:34,242 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:35,512 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:36,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:36,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:38,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:39,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:39,514 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:39,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:41,514 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:41,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:42,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:43,515 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:44,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:44,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:44,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:16:44,391 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:16:44,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:16:44,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:46,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:46,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:47,517 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:48,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:49,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:49,518 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:50,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:50,290 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:16:51,518 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:52,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:52,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:52,878 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:16:52,879 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:16:52,880 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:16:52,880 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:16:53,519 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:16:54,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:54,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:54,519 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:55,505 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:16:55,520 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:57,476 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:16:57,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:57,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:16:57,520 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:59,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:16:59,391 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:16:59,521 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:16:59,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:17:00,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:00,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:00,512 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:02,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:02,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:03,522 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:05,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:05,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:05,523 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:06,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:07,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:07,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:07,524 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:10,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:10,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:11,466 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:11,525 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:12,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:12,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:13,526 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:14,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:17:14,391 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:17:14,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:17:15,527 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:15,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:15,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:16,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:17,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:17,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:19,528 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:20,292 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:17:20,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:20,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:21,529 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:22,116 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:22,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:22,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:23,530 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:25,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:25,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:27,434 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:27,531 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:27,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:27,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:29,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:17:29,392 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:17:29,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:17:29,532 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:30,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:30,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:31,533 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:32,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:32,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:32,757 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:35,535 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:36,018 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:17:36,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:36,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:37,535 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:38,080 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:38,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:38,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:39,536 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:40,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:41,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:41,537 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:43,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:43,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:43,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:44,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:17:44,392 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:17:44,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:17:45,538 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:45,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:45,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:47,539 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:48,740 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:48,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:48,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:49,539 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:50,294 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:17:51,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:51,028 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:53,540 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:53,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:53,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:54,070 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:55,541 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:55,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:55,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:57,542 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:17:59,001 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:59,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:17:59,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:17:59,392 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:17:59,420 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:17:59,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:17:59,585 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:17:59,586 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:17:59,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:17:59,587 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:18:00,543 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:18:00,543 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:01,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:01,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:01,544 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:03,544 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:03,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:03,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:04,718 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:05,545 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:05,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:05,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:09,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:09,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:09,546 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:10,046 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:11,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:11,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:11,547 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:13,549 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:14,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:18:14,392 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:18:14,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:18:14,579 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:18:14,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:14,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:15,382 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:17,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:17,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:17,549 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:19,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:19,550 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:19,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:20,295 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:18:20,710 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:21,551 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:22,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:22,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:24,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:24,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:25,552 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:26,036 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:26,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:26,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:27,552 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:29,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:29,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:29,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:18:29,392 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:18:29,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:18:29,553 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:31,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:31,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:31,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:33,554 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:34,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:34,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:35,555 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:36,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:36,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:36,682 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:37,555 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:39,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:39,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:41,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:41,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:41,556 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:42,005 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:43,557 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:44,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:18:44,392 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:18:44,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:18:44,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:44,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:45,558 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:46,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:46,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:47,086 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:49,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:49,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:49,559 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:18:50,297 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:18:52,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:52,726 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:18:52,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:52,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:55,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:55,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:57,299 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:18:57,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:57,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:18:59,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:18:59,393 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:18:59,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:19:00,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:00,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:02,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:02,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:02,382 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:03,565 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:05,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:05,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:05,565 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:07,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:07,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:07,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:07,697 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:10,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:10,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:11,567 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:12,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:12,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:13,010 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:13,568 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:14,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:19:14,393 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:19:14,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:19:15,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:15,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:15,568 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:17,350 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:19:17,351 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:19:17,352 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:19:17,352 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:19:17,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:17,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:17,569 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:19:18,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:18,569 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:19,570 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:20,299 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:19:20,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:20,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:21,570 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:22,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:22,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:23,571 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:23,634 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:25,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:25,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:27,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:27,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:27,571 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:28,946 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:29,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:19:29,392 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:19:29,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:19:29,573 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:30,795 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:19:30,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:30,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:31,573 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:33,771 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:33,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:34,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:35,575 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:35,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:35,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:37,575 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:38,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:38,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:39,569 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:39,576 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:40,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:40,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:43,577 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:43,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:43,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:44,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:19:44,393 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:19:44,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:19:45,562 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:45,578 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:45,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:45,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:47,578 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:48,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:48,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:50,301 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:19:50,837 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:50,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:51,302 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:51,579 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:53,580 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:53,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:53,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:55,581 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:19:55,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:55,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:56,504 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:19:58,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:58,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:19:59,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:19:59,393 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:19:59,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:19:59,582 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:00,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:00,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:01,582 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:01,819 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:03,584 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:03,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:03,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:05,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:05,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:07,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:07,585 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:09,172 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:20:09,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:09,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:09,585 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:11,586 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:12,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:12,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:12,440 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:14,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:14,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:14,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:20:14,393 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:20:14,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:20:15,587 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:16,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:17,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:17,588 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:17,743 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:19,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:19,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:19,589 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:20,302 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:20:21,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:22,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:23,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:23,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:23,747 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:20:23,748 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:20:23,749 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:20:23,749 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:20:24,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:24,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:24,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:20:24,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:25,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:27,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:27,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:27,591 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:28,385 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:29,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:20:29,393 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:20:29,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:29,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:29,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:20:31,592 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:32,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:32,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:33,593 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:33,700 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:34,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:34,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:35,594 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:37,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:37,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:39,016 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:39,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:39,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:39,594 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:41,595 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:42,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:42,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:43,596 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:44,336 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:44,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:20:44,393 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:20:44,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:20:44,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:44,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:45,596 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:48,155 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:20:48,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:48,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:49,598 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:49,658 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:50,305 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:20:51,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:51,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:51,598 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:53,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:53,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:53,599 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:54,977 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:20:56,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:56,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:57,600 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:20:58,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:58,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:20:59,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:20:59,394 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:20:59,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:20:59,602 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:00,572 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:01,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:01,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:01,602 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:03,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:03,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:05,604 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:05,613 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:06,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:06,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:07,605 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:08,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:08,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:09,605 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:10,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:10,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:10,936 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:13,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:13,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:13,606 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:14,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:21:14,393 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:21:14,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:21:15,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:15,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:15,607 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:16,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:17,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:17,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:17,608 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:19,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:19,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:20,306 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:21:21,609 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:22,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:22,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:22,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:23,610 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:25,354 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:21:25,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:25,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:25,611 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:27,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:28,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:28,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:29,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:21:29,394 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:21:29,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:21:29,612 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:30,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:30,245 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:21:30,247 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:21:30,247 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:21:30,247 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:21:30,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:30,612 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:21:31,615 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:32,616 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:32,866 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:33,130 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:33,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:33,616 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:35,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:35,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:37,617 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:38,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:38,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:38,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:39,618 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:40,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:40,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:41,618 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:43,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:43,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:43,483 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:44,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:21:44,394 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:21:44,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:21:45,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:45,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:45,619 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:47,620 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:48,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:48,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:48,812 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:49,620 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:50,308 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:21:50,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:50,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:53,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:53,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:53,622 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:54,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:55,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:55,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:55,622 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:57,624 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:21:58,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:58,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:21:59,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:21:59,394 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:21:59,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:21:59,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:22:00,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:00,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:01,625 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:03,625 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:03,800 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:22:03,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:03,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:04,766 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:05,626 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:06,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:06,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:08,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:08,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:09,628 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:10,082 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:11,629 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:11,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:11,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:13,630 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:13,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:13,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:14,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:22:14,394 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:22:14,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:22:15,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:16,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:16,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:17,631 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:18,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:18,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:19,632 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:20,309 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:22:20,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:21,632 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:21,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:21,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:23,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:23,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:25,633 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:26,026 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:26,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:26,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:27,634 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:28,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:28,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:29,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:22:29,394 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:22:29,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:22:29,634 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:30,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:30,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:31,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:33,636 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:33,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:33,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:35,636 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:35,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:35,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:36,670 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:36,696 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:22:36,698 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:22:36,698 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:22:36,699 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:22:37,637 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:22:37,637 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:38,637 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:38,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:38,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:41,638 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:41,992 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:42,333 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:22:42,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:42,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:43,639 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:44,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:44,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:44,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:22:44,394 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:22:44,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:22:45,640 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:47,313 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:47,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:47,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:49,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:49,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:49,641 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:50,311 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:22:51,642 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:52,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:52,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:52,630 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:53,643 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:54,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:54,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:57,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:57,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:57,645 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:22:57,938 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:22:59,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:22:59,394 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:22:59,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:59,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:22:59,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:22:59,645 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:01,646 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:02,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:02,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:03,252 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:04,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:04,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:05,647 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:07,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:07,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:07,648 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:08,576 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:09,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:09,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:09,649 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:12,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:12,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:13,649 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:13,890 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:14,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:23:14,395 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:23:14,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:23:14,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:14,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:15,650 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:16,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:16,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:17,650 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:19,202 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:19,651 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:19,796 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:23:19,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:19,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:20,313 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:23:22,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:22,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:23,652 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:24,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:24,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:24,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:25,653 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:27,654 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:27,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:27,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:29,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:23:29,395 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:23:29,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:23:29,572 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:29,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:29,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:31,655 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:31,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:31,940 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:33,655 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:34,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:34,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:35,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:35,656 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:36,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:36,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:39,658 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:39,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:39,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:40,477 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:41,658 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:41,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:41,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:43,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:23:43,160 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:23:43,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:23:43,160 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:23:43,659 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:23:43,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:43,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:43,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:44,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:23:44,395 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:23:44,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:23:44,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:45,591 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:46,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:46,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:47,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:48,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:48,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:49,661 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:50,314 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:23:51,103 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:51,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:51,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:51,661 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:53,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:53,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:55,663 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:56,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:23:57,220 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:23:57,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:57,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:23:57,663 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:23:59,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:23:59,395 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:23:59,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:23:59,663 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:00,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:00,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:01,578 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:02,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:02,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:03,665 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:04,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:04,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:05,666 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:06,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:06,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:07,037 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:07,666 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:08,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:08,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:11,667 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:12,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:12,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:12,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:13,668 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:14,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:14,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:14,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:24:14,395 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:24:14,534 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:24:15,669 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:17,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:17,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:17,673 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:19,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:19,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:19,670 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:20,315 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:24:21,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:21,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:21,671 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:22,982 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:23,671 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:23,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:23,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:26,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:26,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:27,673 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:28,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:28,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:28,295 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:29,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:24:29,395 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:24:29,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:24:29,673 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:30,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:30,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:31,674 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:33,608 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:34,462 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:24:34,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:34,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:35,675 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:36,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:36,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:37,676 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:38,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:39,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:39,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:39,677 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:41,657 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:41,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:43,679 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:44,254 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:44,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:24:44,395 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:24:44,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:24:44,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:44,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:45,679 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:46,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:46,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:47,680 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:49,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:49,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:49,566 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:49,592 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:24:49,594 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:24:49,594 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:24:49,595 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:24:49,680 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:24:50,317 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:24:50,681 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:51,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:51,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:51,681 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:53,682 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:54,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:54,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:54,877 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:24:55,682 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:24:56,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:56,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:58,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:58,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:24:59,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:24:59,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:24:59,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:24:59,684 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:00,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:01,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:01,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:01,684 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:03,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:03,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:03,685 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:05,492 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:06,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:06,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:07,686 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:08,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:08,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:09,687 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:10,809 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:11,688 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:11,992 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:25:12,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:12,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:14,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:25:14,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:25:14,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:25:14,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:14,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:15,689 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:16,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:16,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:16,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:17,690 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:18,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:18,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:19,690 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:20,319 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:25:21,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:21,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:21,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:23,691 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:23,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:23,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:25,692 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:26,757 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:26,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:26,800 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:27,692 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:28,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:28,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:29,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:25:29,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:25:29,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:25:31,694 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:31,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:31,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:32,084 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:33,694 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:34,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:34,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:35,695 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:36,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:36,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:37,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:38,696 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:38,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:39,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:41,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:41,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:42,697 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:42,706 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:43,956 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:43,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:44,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:25:44,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:25:44,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:25:44,697 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:45,989 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:45,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:46,698 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:48,021 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:49,328 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:25:49,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:49,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:50,320 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:25:50,700 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:52,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:52,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:52,701 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:53,347 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:54,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:54,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:54,701 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:56,027 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:25:56,028 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:25:56,029 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:25:56,029 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:25:56,702 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:25:56,702 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:57,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:57,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:58,657 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:25:58,703 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:25:59,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:59,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:25:59,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:25:59,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:25:59,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:26:00,703 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:02,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:02,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:02,704 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:03,972 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:04,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:04,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:06,705 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:07,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:07,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:08,706 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:09,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:09,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:09,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:10,706 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:12,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:12,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:14,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:14,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:14,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:26:14,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:26:14,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:26:14,564 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:14,707 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:16,708 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:17,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:17,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:18,708 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:19,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:19,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:19,930 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:20,321 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:26:22,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:22,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:22,710 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:24,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:24,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:24,711 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:25,243 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:26,711 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:27,882 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:26:27,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:27,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:29,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:26:29,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:26:29,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:26:30,572 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:30,713 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:30,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:30,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:32,714 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:32,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:32,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:34,715 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:35,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:35,887 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:35,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:37,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:37,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:38,716 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:40,717 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:40,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:40,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:41,224 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:42,718 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:42,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:42,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:44,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:26:44,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:26:44,536 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:26:46,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:46,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:46,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:46,719 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:48,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:48,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:48,719 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:50,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:50,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:50,323 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:26:50,720 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:51,854 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:52,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:52,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:54,721 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:54,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:54,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:56,721 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:57,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:57,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:57,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:26:58,722 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:26:59,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:26:59,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:26:59,536 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:26:59,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:26:59,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:01,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:01,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:02,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:02,515 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:27:02,516 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:27:02,517 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:27:02,517 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:27:02,724 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:27:02,724 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:04,725 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:05,341 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:27:05,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:05,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:06,725 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:07,805 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:08,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:08,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:10,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:10,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:10,726 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:12,727 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:13,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:13,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:13,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:14,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:27:14,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:27:14,536 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:27:14,728 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:15,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:15,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:18,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:18,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:18,434 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:18,729 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:20,325 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:27:20,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:20,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:20,729 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:22,730 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:23,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:23,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:23,746 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:25,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:25,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:26,731 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:28,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:28,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:28,732 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:29,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:29,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:27:29,396 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:27:29,536 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:27:30,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:30,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:30,733 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:33,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:33,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:34,379 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:34,734 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:35,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:35,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:36,734 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:38,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:38,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:38,735 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:39,692 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:40,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:40,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:42,736 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:43,816 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:27:43,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:43,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:44,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:27:44,397 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:27:44,536 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:27:44,737 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:45,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:46,737 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:46,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:46,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:48,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:48,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:50,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:50,326 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:27:50,738 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:51,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:51,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:52,739 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:53,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:53,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:54,739 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:55,639 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:27:56,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:56,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:58,741 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:27:58,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:58,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:27:59,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:27:59,397 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:27:59,536 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:28:00,741 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:00,950 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:01,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:01,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:02,742 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:03,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:03,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:04,742 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:06,258 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:06,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:06,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:08,744 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:08,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:08,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:08,948 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:28:08,950 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:28:08,950 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:28:08,950 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:28:09,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:28:10,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:11,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:11,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:11,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:11,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:12,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:13,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:13,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:14,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:28:14,397 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:28:14,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:28:16,746 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:16,899 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:16,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:16,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:18,747 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:18,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:18,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:20,328 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:28:20,747 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:22,222 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:22,302 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:28:22,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:22,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:24,749 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:25,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:25,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:26,750 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:27,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:27,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:27,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:28,750 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:29,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:28:29,397 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:28:29,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:28:30,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:30,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:32,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:32,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:32,751 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:32,852 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:34,752 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:35,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:35,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:36,752 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:37,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:37,203 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:38,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:40,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:40,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:40,753 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:42,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:42,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:42,754 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:43,481 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:44,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:28:44,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:28:44,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:28:44,754 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:45,234 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:45,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:47,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:47,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:48,756 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:48,793 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:49,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:49,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:50,330 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:28:50,757 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:28:51,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:51,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:54,332 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:54,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:54,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:56,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:56,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:59,332 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:28:59,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:28:59,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:28:59,482 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:28:59,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:59,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:28:59,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:29:02,234 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:02,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:04,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:04,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:04,760 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:04,846 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:06,761 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:07,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:07,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:09,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:09,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:10,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:10,762 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:12,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:12,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:12,762 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:14,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:29:14,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:29:14,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:29:14,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:14,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:14,763 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:15,563 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:17,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:17,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:18,765 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:19,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:19,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:20,332 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:29:20,765 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:21,333 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:22,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:22,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:22,766 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:24,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:24,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:26,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:26,736 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:29:26,738 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:29:26,738 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:29:26,739 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:29:26,767 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:29:26,767 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:27,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:27,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:28,767 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:29,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:29:29,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:29:29,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:29:29,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:29,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:29,768 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:30,768 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:32,027 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:32,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:32,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:34,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:34,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:34,770 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:36,770 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:37,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:38,191 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:29:38,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:38,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:38,771 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:41,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:41,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:42,643 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:42,772 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:43,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:43,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:44,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:29:44,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:29:44,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:29:44,773 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:46,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:46,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:46,774 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:47,950 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:48,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:48,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:50,334 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:29:50,775 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:51,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:51,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:52,775 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:53,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:53,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:53,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:54,776 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:56,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:56,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:58,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:58,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:29:58,572 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:29:58,777 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:29:59,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:29:59,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:29:59,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:30:00,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:00,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:00,778 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:02,778 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:02,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:03,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:03,871 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:05,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:05,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:06,780 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:07,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:07,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:08,780 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:09,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:09,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:09,879 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:10,781 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:11,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:11,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:14,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:30:14,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:30:14,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:30:14,626 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:14,782 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:15,439 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:30:15,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:15,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:16,782 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:18,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:18,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:18,783 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:19,789 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:20,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:20,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:20,336 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:30:20,784 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:23,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:23,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:24,785 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:25,107 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:25,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:25,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:26,786 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:28,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:28,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:28,786 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:29,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:30:29,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:30:29,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:30:30,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:30,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:30,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:32,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:32,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:32,787 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:33,096 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:30:33,097 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:30:33,097 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:30:33,098 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:30:33,788 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:30:34,788 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:35,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:35,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:35,727 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:35,789 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:36,789 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:37,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:37,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:40,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:40,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:40,790 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:41,036 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:42,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:42,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:42,791 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:44,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:30:44,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:30:44,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:30:44,791 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:45,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:45,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:46,351 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:47,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:47,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:48,793 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:50,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:50,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:50,337 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:30:50,793 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:51,660 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:52,794 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:53,772 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:30:53,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:53,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:55,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:55,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:56,795 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:56,969 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:30:58,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:58,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:30:58,796 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:30:59,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:30:59,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:30:59,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:31:00,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:00,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:00,797 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:02,277 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:03,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:03,693 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:04,798 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:05,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:05,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:06,799 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:07,578 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:08,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:08,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:08,799 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:10,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:10,755 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:12,800 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:12,882 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:13,767 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:13,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:14,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:31:14,398 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:31:14,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:31:14,801 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:15,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:15,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:16,801 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:18,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:18,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:18,865 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:20,339 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:31:20,803 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:20,884 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:20,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:22,803 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:23,501 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:24,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:24,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:24,804 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:26,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:26,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:28,805 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:28,812 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:29,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:29,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:29,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:31:29,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:31:29,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:31:30,805 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:32,525 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:31:32,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:32,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:32,806 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:34,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:34,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:36,810 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:37,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:37,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:38,811 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:39,428 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:39,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:39,455 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:31:39,457 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:31:39,457 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:31:39,457 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:31:39,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:39,811 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:31:40,811 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:41,812 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:42,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:42,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:44,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:31:44,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:31:44,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:44,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:44,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:31:44,612 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:44,812 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:46,813 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:47,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:47,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:48,814 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:49,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:49,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:50,047 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:50,340 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:31:52,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:52,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:52,816 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:54,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:54,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:54,816 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:55,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:31:56,817 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:31:57,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:57,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:59,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:31:59,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:31:59,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:59,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:31:59,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:32:00,556 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:00,819 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:02,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:02,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:02,819 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:04,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:04,419 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:04,820 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:05,979 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:07,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:07,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:08,821 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:10,822 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:11,120 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:32:11,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:11,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:11,288 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:12,822 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:14,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:14,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:14,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:32:14,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:32:14,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:32:14,823 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:16,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:16,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:16,573 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:18,824 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:19,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:19,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:20,342 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:32:20,825 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:21,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:21,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:21,908 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:22,825 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:24,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:24,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:26,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:26,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:26,827 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:27,209 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:28,827 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:29,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:32:29,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:32:29,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:29,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:29,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:32:30,828 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:31,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:31,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:32,509 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:34,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:34,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:34,829 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:36,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:36,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:36,830 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:37,815 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:38,830 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:39,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:39,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:41,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:41,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:42,831 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:43,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:43,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:43,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:44,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:32:44,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:32:44,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:32:44,832 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:45,801 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:32:45,803 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:32:45,803 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:32:45,803 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:32:45,832 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:32:46,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:46,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:46,832 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:47,833 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:48,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:49,856 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:32:49,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:49,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:50,343 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:32:50,834 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:51,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:51,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:52,834 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:53,741 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:54,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:54,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:54,835 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:56,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:56,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:58,836 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:32:59,043 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:32:59,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:32:59,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:32:59,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:32:59,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:32:59,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:00,837 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:01,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:01,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:02,837 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:04,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:04,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:05,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:06,838 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:07,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:07,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:08,839 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:09,661 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:10,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:10,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:10,840 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:12,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:12,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:14,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:33:14,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:33:14,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:33:14,841 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:14,971 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:15,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:15,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:16,842 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:17,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:17,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:18,842 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:20,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:20,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:20,272 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:20,345 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:33:22,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:22,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:22,843 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:24,844 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:25,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:25,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:25,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:26,845 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:28,642 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:33:28,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:28,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:29,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:33:29,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:33:29,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:33:30,618 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:30,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:30,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:30,846 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:32,847 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:33,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:33,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:34,847 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:35,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:35,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:36,196 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:38,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:38,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:38,849 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:40,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:40,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:40,850 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:41,507 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:42,850 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:43,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:43,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:44,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:33:44,399 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:33:44,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:33:45,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:45,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:46,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:46,851 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:48,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:48,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:48,852 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:50,346 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:33:50,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:50,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:50,853 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:52,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:52,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:33:52,151 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:33:52,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:33:52,152 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:33:52,853 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:33:53,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:53,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:53,854 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:54,854 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:55,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:55,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:56,855 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:57,443 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:33:58,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:58,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:33:58,855 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:33:59,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:33:59,400 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:33:59,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:34:00,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:00,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:00,856 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:02,758 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:03,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:03,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:04,857 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:06,857 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:07,265 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:34:07,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:07,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:08,063 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:08,858 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:09,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:10,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:12,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:12,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:12,859 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:13,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:14,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:34:14,400 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:34:14,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:34:14,860 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:15,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:15,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:16,860 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:17,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:17,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:18,680 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:20,348 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:34:20,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:20,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:20,862 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:22,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:22,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:22,862 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:23,987 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:24,863 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:25,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:25,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:27,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:27,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:28,864 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:29,288 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:29,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:34:29,400 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:34:29,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:34:30,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:30,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:30,865 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:32,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:32,643 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:32,865 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:34,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:35,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:35,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:36,867 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:37,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:37,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:38,867 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:39,912 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:40,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:40,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:40,868 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:42,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:42,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:44,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:34:44,400 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:34:44,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:34:44,869 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:45,220 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:46,097 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:34:46,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:46,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:46,870 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:48,870 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:49,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:49,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:50,349 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:34:50,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:51,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:51,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:52,872 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:54,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:54,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:54,873 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:55,835 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:34:56,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:56,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:56,873 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:34:58,517 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:34:58,518 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:34:58,519 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:34:58,519 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:34:58,873 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:34:59,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:59,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:34:59,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:34:59,400 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:34:59,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:34:59,874 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:00,874 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:01,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:01,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:01,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:02,875 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:04,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:04,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:04,876 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:06,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:06,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:06,457 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:08,878 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:09,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:09,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:10,878 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:11,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:11,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:11,768 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:12,879 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:14,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:14,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:35:14,400 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:35:14,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:14,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:35:16,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:16,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:16,880 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:17,079 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:18,881 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:19,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:19,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:20,351 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:35:20,881 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:21,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:21,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:22,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:24,882 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:24,910 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:35:24,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:24,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:26,883 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:27,689 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:28,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:28,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:28,884 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:29,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:35:29,400 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:35:29,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:35:30,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:30,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:32,890 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:32,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:33,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:33,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:34,891 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:35,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:35,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:36,892 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:38,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:38,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:38,310 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:40,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:40,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:40,893 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:42,894 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:43,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:43,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:43,617 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:44,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:35:44,401 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:35:44,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:35:44,895 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:45,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:45,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:46,895 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:47,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:47,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:48,931 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:49,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:50,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:50,352 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:35:50,897 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:52,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:52,046 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:52,897 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:54,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:54,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:54,237 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:35:54,898 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:56,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:56,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:58,899 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:35:58,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:58,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:35:59,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:35:59,401 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:35:59,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:35:59,617 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:00,900 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:02,433 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:36:02,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:02,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:02,901 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:04,849 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:04,876 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:36:04,878 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:36:04,878 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:36:04,878 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:36:04,902 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:36:05,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:05,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:05,902 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:06,902 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:07,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:07,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:08,903 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:10,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:10,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:10,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:10,903 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:12,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:12,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:14,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:36:14,401 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:36:14,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:36:14,905 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:15,452 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:15,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:15,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:16,906 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:17,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:17,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:18,906 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:20,354 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:36:20,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:20,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:20,756 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:22,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:22,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:22,907 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:24,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:24,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:24,909 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:26,063 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:26,910 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:27,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:27,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:29,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:29,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:29,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:36:29,401 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:36:29,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:36:30,911 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:31,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:32,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:32,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:32,912 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:34,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:34,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:34,912 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:36,685 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:37,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:37,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:38,913 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:40,866 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:36:40,884 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:40,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:40,914 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:41,986 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:42,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:42,915 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:42,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:44,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:36:44,401 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:36:44,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:36:45,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:45,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:46,916 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:47,289 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:47,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:47,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:48,917 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:50,356 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:36:50,917 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:50,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:50,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:52,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:52,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:52,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:54,919 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:56,001 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:56,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:56,919 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:57,906 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:36:58,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:58,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:36:58,920 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:36:59,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:36:59,401 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:36:59,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:37:01,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:01,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:02,922 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:03,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:03,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:03,216 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:04,922 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:05,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:05,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:06,923 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:08,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:08,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:08,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:10,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:10,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:10,925 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:11,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:37:11,196 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:37:11,196 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:37:11,197 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:37:11,925 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:37:12,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:12,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:12,925 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:13,826 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:13,926 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:14,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:37:14,401 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:37:14,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:37:14,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:14,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:14,926 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:18,511 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:37:18,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:18,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:18,927 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:19,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:20,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:37:20,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:20,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:20,928 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:22,929 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:23,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:23,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:24,450 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:25,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:25,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:26,930 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:27,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:27,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:28,930 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:29,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:37:29,401 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:37:29,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:37:29,624 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:30,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:30,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:30,932 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:32,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:32,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:32,933 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:35,058 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:35,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:35,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:36,934 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:37,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:37,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:38,935 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:39,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:39,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:40,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:40,935 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:42,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:42,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:44,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:44,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:44,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:37:44,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:37:44,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:37:44,937 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:45,593 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:46,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:46,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:46,937 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:48,938 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:48,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:48,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:50,359 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:37:50,982 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:50,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:51,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:52,939 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:54,463 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:37:54,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:54,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:54,940 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:56,289 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:37:56,941 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:37:57,419 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:57,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:59,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:37:59,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:37:59,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:59,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:37:59,544 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:38:00,942 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:01,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:02,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:02,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:02,943 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:04,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:04,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:04,944 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:06,915 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:07,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:07,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:08,945 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:09,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:09,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:10,946 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:12,225 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:12,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:12,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:12,947 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:14,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:38:14,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:38:14,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:38:14,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:14,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:16,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:16,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:16,948 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:17,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:17,564 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:38:17,565 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:38:17,565 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:38:17,566 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:38:17,948 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:38:18,948 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:19,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:19,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:19,949 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:20,361 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:38:20,949 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:21,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:21,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:22,847 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:24,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:24,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:24,950 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:26,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:26,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:26,951 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:28,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:28,952 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:29,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:38:29,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:38:29,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:38:29,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:29,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:32,953 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:33,119 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:38:33,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:33,152 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:33,467 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:34,953 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:35,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:35,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:36,954 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:38,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:38,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:38,783 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:40,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:40,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:40,955 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:42,956 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:43,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:43,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:44,097 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:44,401 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:38:44,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:38:44,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:38:44,956 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:45,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:45,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:48,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:48,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:48,958 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:50,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:50,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:50,362 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:38:50,958 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:38:52,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:52,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:54,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:54,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:54,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:57,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:57,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:59,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:38:59,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:38:59,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:59,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:38:59,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:38:59,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:39:02,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:02,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:04,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:04,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:04,964 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:05,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:06,965 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:06,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:06,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:08,966 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:10,500 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:10,519 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:39:10,541 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:10,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:12,967 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:13,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:13,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:14,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:39:14,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:39:14,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:39:14,968 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:15,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:15,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:15,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:16,969 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:18,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:18,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:20,364 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:39:20,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:20,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:20,970 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:21,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:22,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:22,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:22,971 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:24,972 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:25,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:25,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:26,421 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:27,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:27,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:28,973 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:29,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:39:29,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:39:29,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:39:30,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:30,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:30,974 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:31,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:32,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:32,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:32,975 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:35,050 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:39:35,051 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:39:35,051 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:39:35,052 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:39:35,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:35,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:35,976 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:39:35,976 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:36,976 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:37,053 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:37,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:37,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:38,977 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:39,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:39,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:40,978 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:42,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:42,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:42,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:44,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:39:44,402 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:39:44,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:39:44,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:44,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:44,979 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:46,980 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:47,633 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:48,183 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:39:48,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:48,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:48,980 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:50,366 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:39:51,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:51,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:52,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:52,982 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:53,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:53,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:54,982 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:56,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:56,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:56,983 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:39:58,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:58,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:39:58,243 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:39:59,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:39:59,403 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:39:59,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:40:00,985 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:01,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:01,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:02,986 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:03,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:03,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:03,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:04,986 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:05,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:05,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:07,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:08,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:08,853 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:08,987 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:10,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:10,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:10,988 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:12,989 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:13,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:13,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:14,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:14,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:40:14,403 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:40:14,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:40:15,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:15,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:16,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:18,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:18,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:18,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:19,453 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:20,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:20,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:20,367 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:40:20,992 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:22,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:22,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:24,761 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:24,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:25,428 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:40:25,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:25,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:26,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:28,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:28,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:28,995 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:29,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:40:29,403 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:40:29,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:40:30,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:30,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:30,598 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:33,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:33,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:33,998 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:35,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:35,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:35,999 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:36,027 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:37,999 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:38,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:38,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:40,000 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:40,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:40,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:41,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:41,369 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:40:41,370 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:40:41,370 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:40:41,371 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:40:42,001 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:40:43,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:43,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:44,002 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:44,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:40:44,403 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:40:44,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:40:45,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:45,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:46,002 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:46,616 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:48,003 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:48,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:48,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:50,369 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:40:50,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:50,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:51,953 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:52,004 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:53,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:53,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:54,005 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:55,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:55,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:56,005 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:40:57,267 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:40:58,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:58,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:40:59,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:40:59,403 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:40:59,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:41:00,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:00,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:00,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:02,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:02,574 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:04,008 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:04,267 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:41:04,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:04,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:07,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:07,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:07,881 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:08,009 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:09,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:09,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:10,011 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:12,011 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:12,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:12,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:13,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:14,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:14,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:14,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:41:14,403 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:41:14,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:41:16,012 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:16,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:16,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:18,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:18,501 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:18,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:18,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:20,014 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:20,370 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:41:20,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:20,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:23,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:23,737 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:23,806 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:24,015 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:25,757 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:25,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:26,016 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:28,016 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:28,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:28,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:29,111 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:29,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:41:29,403 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:41:29,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:41:30,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:30,837 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:32,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:33,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:33,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:34,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:34,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:35,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:35,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:36,019 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:37,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:37,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:39,713 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:40,021 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:41,161 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:41:41,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:41,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:42,021 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:44,022 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:44,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:44,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:44,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:41:44,404 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:41:44,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:41:45,017 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:46,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:46,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:47,691 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:41:47,692 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:41:47,693 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:41:47,693 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:41:48,023 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:41:48,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:49,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:49,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:50,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:50,313 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:50,372 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:41:51,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:51,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:52,025 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:54,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:54,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:55,614 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:41:56,027 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:56,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:56,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:58,027 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:41:58,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:58,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:41:59,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:41:59,403 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:41:59,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:42:00,028 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:00,921 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:00,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:00,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:03,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:03,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:04,030 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:05,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:06,001 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:06,031 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:06,229 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:08,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:08,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:08,032 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:11,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:11,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:11,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:12,033 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:13,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:13,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:14,034 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:14,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:42:14,404 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:42:14,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:42:16,035 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:16,100 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:16,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:16,587 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:19,670 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:42:19,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:19,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:20,036 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:20,373 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:42:21,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:21,732 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:22,037 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:22,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:24,038 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:24,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:24,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:26,038 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:26,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:26,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:27,458 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:29,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:42:29,404 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:42:29,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:42:29,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:29,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:30,040 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:31,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:31,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:32,041 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:32,760 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:34,042 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:34,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:34,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:36,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:36,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:38,043 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:38,064 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:38,884 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:38,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:40,044 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:41,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:41,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:42,044 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:43,371 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:43,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:43,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:44,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:42:44,404 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:42:44,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:42:46,046 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:46,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:46,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:48,047 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:48,685 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:48,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:48,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:50,047 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:50,375 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:42:50,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:50,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:53,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:53,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:53,996 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:54,022 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:42:54,024 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:42:54,024 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:42:54,024 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:42:54,049 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:42:54,049 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:56,049 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:57,149 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:42:57,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:57,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:42:58,050 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:42:59,313 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:42:59,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:42:59,405 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:42:59,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:43:00,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:00,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:02,051 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:02,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:02,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:04,052 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:04,619 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:05,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:05,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:06,053 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:07,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:07,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:09,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:09,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:09,925 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:10,054 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:12,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:12,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:12,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:14,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:14,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:14,235 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:14,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:43:14,405 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:43:14,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:43:15,232 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:16,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:16,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:18,056 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:19,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:19,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:20,057 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:20,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:43:20,379 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:21,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:21,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:22,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:24,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:24,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:25,841 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:26,059 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:26,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:26,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:28,060 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:29,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:29,203 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:29,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:43:29,405 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:43:29,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:43:30,060 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:31,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:31,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:31,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:34,062 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:34,741 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:43:34,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:34,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:36,062 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:36,449 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:37,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:37,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:38,063 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:39,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:39,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:41,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:41,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:41,754 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:42,064 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:44,065 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:44,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:43:44,405 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:43:44,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:44,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:44,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:43:46,066 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:46,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:46,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:47,059 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:49,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:49,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:50,067 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:50,378 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:43:51,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:51,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:52,068 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:52,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:54,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:54,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:54,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:56,777 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:56,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:57,675 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:43:58,070 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:43:59,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:43:59,405 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:43:59,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:43:59,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:43:59,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:00,070 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:00,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:44:00,360 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:44:00,360 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:44:00,360 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:44:01,071 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:44:01,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:01,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:02,071 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:02,988 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:04,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:04,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:06,073 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:06,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:06,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:08,073 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:08,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:09,777 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:09,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:10,074 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:12,075 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:13,341 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:44:13,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:13,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:13,603 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:14,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:44:14,405 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:44:14,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:44:15,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:15,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:16,076 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:18,077 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:18,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:18,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:18,910 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:20,077 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:20,379 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:44:20,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:20,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:23,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:23,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:24,078 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:24,217 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:25,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:25,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:26,079 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:28,080 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:28,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:28,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:29,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:44:29,406 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:44:29,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:44:29,603 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:30,467 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:30,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:32,081 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:32,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:32,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:34,082 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:34,825 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:35,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:35,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:36,083 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:37,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:37,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:40,084 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:40,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:40,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:40,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:42,085 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:42,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:42,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:44,085 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:44,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:44:44,406 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:44:44,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:44:45,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:45,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:45,431 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:47,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:47,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:48,087 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:50,088 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:50,380 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:44:50,714 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:44:50,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:50,743 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:50,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:52,090 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:53,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:53,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:55,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:55,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:56,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:44:56,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:58,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:44:58,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:58,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:44:59,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:44:59,406 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:44:59,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:45:00,092 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:00,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:00,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:01,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:03,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:03,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:04,094 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:05,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:05,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:06,095 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:06,676 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:06,703 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:45:06,704 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:45:06,705 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:45:06,705 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:45:07,095 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:45:08,095 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:08,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:08,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:10,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:10,718 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:11,981 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:12,097 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:13,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:13,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:14,098 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:14,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:45:14,406 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:45:14,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:45:15,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:15,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:16,098 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:17,291 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:18,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:18,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:20,100 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:20,382 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:45:20,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:20,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:22,100 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:22,384 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:23,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:23,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:24,101 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:25,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:25,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:27,909 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:28,102 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:29,120 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:45:29,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:29,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:29,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:45:29,406 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:45:29,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:45:30,103 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:31,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:31,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:32,104 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:33,213 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:34,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:34,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:36,105 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:37,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:37,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:38,106 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:38,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:39,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:39,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:40,107 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:42,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:42,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:43,823 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:44,108 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:44,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:44,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:44,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:45:44,406 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:45:44,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:45:46,109 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:47,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:47,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:48,110 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:49,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:49,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:50,384 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:45:52,111 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:52,467 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:52,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:54,112 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:54,439 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:45:54,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:54,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:56,112 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:57,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:57,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:58,113 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:45:59,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:45:59,406 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:45:59,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:59,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:45:59,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:45:59,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:02,114 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:02,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:02,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:04,115 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:04,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:04,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:05,047 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:06,116 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:07,847 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:46:07,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:07,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:10,117 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:10,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:10,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:10,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:12,118 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:12,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:12,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:13,032 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:46:13,033 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:46:13,033 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:46:13,033 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:46:13,118 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:46:14,119 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:14,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:46:14,406 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:46:14,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:46:15,633 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:15,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:15,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:17,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:17,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:18,121 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:20,121 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:20,385 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:46:20,966 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:21,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:21,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:22,122 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:23,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:23,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:26,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:26,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:26,124 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:26,270 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:28,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:28,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:28,125 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:29,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:46:29,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:46:29,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:46:30,125 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:30,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:30,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:31,576 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:32,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:32,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:34,126 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:36,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:36,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:36,127 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:36,874 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:38,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:38,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:38,128 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:41,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:41,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:42,129 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:42,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:43,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:43,234 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:44,130 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:44,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:46:44,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:46:44,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:46:46,131 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:46,601 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:46:46,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:46,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:47,491 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:49,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:49,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:50,132 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:50,387 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:46:51,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:51,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:52,133 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:52,796 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:53,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:53,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:54,134 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:56,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:56,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:58,097 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:46:58,135 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:46:58,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:58,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:46:59,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:46:59,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:46:59,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:47:00,136 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:01,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:01,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:02,137 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:03,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:03,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:03,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:06,139 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:06,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:06,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:08,139 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:08,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:08,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:08,698 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:10,140 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:11,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:11,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:13,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:13,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:14,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:14,141 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:14,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:47:14,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:47:14,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:47:16,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:16,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:16,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:18,143 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:18,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:18,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:19,310 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:19,336 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:47:19,337 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:47:19,338 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:47:19,338 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:47:20,144 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:47:20,389 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:47:21,144 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:21,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:21,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:22,145 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:24,145 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:24,612 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:25,207 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:47:25,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:25,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:26,146 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:27,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:27,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:29,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:47:29,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:47:29,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:47:29,917 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:30,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:30,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:30,147 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:32,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:32,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:32,148 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:34,149 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:34,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:34,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:35,223 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:36,150 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:36,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:36,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:39,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:39,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:40,151 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:40,533 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:42,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:42,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:42,152 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:44,152 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:44,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:47:44,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:47:44,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:47:44,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:45,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:45,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:47,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:47,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:48,154 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:50,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:50,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:50,155 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:50,391 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:47:51,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:52,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:52,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:52,155 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:55,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:55,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:56,157 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:56,461 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:47:57,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:57,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:47:58,157 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:47:59,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:47:59,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:47:59,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:48:00,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:00,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:00,158 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:01,597 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:03,845 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:48:03,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:03,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:04,159 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:06,160 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:06,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:06,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:07,081 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:08,161 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:08,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:08,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:11,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:12,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:12,162 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:12,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:14,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:14,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:14,163 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:14,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:48:14,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:48:14,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:48:16,164 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:16,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:16,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:17,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:19,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:19,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:20,165 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:20,392 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:48:22,166 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:22,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:22,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:23,014 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:24,167 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:24,280 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:24,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:25,689 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:48:25,690 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:48:25,690 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:48:25,691 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:48:26,168 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:48:27,168 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:27,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:27,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:28,169 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:28,316 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:29,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:29,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:29,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:48:29,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:48:29,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:48:30,169 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:32,170 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:32,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:32,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:33,625 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:34,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:34,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:36,171 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:37,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:37,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:38,171 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:38,939 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:39,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:39,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:40,172 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:43,048 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:48:43,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:43,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:44,173 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:44,245 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:44,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:48:44,407 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:48:44,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:48:45,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:45,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:46,174 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:48,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:48,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:48,175 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:49,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:50,175 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:48:50,394 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:48:50,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:50,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:52,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:52,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:54,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:55,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:55,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:57,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:57,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:48:59,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:48:59,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:48:59,408 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:48:59,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:49:00,040 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:00,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:02,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:02,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:04,600 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:04,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:04,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:06,181 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:07,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:07,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:08,182 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:09,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:09,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:09,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:10,182 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:11,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:11,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:14,184 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:14,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:49:14,408 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:49:14,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:49:14,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:14,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:15,599 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:16,185 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:17,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:17,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:18,185 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:20,389 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:49:20,396 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:49:20,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:20,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:21,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:22,186 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:23,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:23,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:24,187 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:25,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:25,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:26,188 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:26,813 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:28,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:28,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:29,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:49:29,408 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:49:29,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:49:30,192 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:30,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:30,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:32,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:32,192 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:33,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:33,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:34,193 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:35,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:35,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:36,194 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:37,422 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:38,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:38,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:40,195 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:40,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:40,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:42,196 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:42,727 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:43,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:43,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:43,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:49:43,408 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:49:43,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:49:43,409 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:49:44,197 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:49:44,197 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:44,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:49:44,408 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:49:44,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:49:45,197 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:45,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:45,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:47,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:47,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:48,034 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:48,198 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:50,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:50,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:50,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:50,397 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:49:52,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:52,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:52,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:53,334 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:55,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:55,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:56,201 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:58,202 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:49:58,638 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:49:58,822 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:49:58,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:58,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:49:59,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:49:59,408 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:49:59,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:50:00,203 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:00,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:00,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:03,652 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:03,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:03,934 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:04,209 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:05,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:05,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:06,210 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:08,210 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:08,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:08,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:09,232 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:10,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:10,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:12,212 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:13,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:13,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:14,213 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:14,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:50:14,408 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:50:14,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:14,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:50:15,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:15,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:16,214 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:18,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:18,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:19,846 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:20,215 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:20,399 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:50:20,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:20,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:22,216 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:23,718 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:23,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:24,217 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:25,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:25,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:25,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:28,219 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:28,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:28,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:29,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:50:29,408 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:50:29,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:50:30,219 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:30,630 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:30,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:30,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:32,220 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:32,693 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:32,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:35,750 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:35,916 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:50:35,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:35,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:36,221 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:38,222 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:38,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:38,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:40,223 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:40,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:40,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:41,053 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:43,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:43,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:44,224 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:44,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:50:44,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:50:44,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:50:45,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:45,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:46,225 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:46,596 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:48,226 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:48,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:48,902 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:49,693 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:50:49,694 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:50:49,694 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:50:49,695 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:50:50,226 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:50:50,401 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:50:50,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:50,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:51,227 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:52,227 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:52,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:53,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:53,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:54,229 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:55,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:55,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:56,229 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:50:57,625 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:50:58,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:58,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:50:59,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:50:59,408 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:50:59,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:51:00,231 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:00,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:00,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:02,232 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:02,926 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:03,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:03,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:04,232 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:05,935 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:05,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:08,228 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:08,233 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:08,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:08,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:10,234 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:10,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:10,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:12,235 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:13,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:14,236 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:14,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:51:14,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:51:14,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:51:14,581 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:51:14,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:14,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:17,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:17,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:18,237 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:18,831 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:19,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:19,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:20,238 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:20,402 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:51:22,239 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:22,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:22,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:24,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:24,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:24,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:26,240 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:27,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:27,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:28,241 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:29,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:51:29,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:51:29,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:51:29,615 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:29,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:29,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:32,242 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:32,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:32,718 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:34,243 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:34,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:34,755 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:34,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:36,245 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:37,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:37,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:38,245 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:39,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:39,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:40,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:42,246 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:42,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:42,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:44,247 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:44,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:51:44,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:51:44,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:51:44,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:44,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:45,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:46,248 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:47,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:47,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:49,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:49,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:50,249 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:50,404 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:51:50,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:52,250 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:53,434 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:51:53,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:53,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:54,251 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:55,963 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:51:55,990 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:51:55,991 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:51:55,991 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:51:55,991 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:51:56,252 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:51:56,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:56,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:57,252 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:58,253 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:51:58,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:58,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:51:59,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:51:59,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:51:59,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:52:00,253 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:01,277 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:01,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:01,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:02,254 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:03,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:03,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:06,255 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:06,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:06,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:06,584 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:08,256 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:08,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:08,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:10,257 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:11,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:11,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:11,890 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:13,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:13,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:14,258 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:14,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:52:14,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:52:14,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:52:16,259 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:16,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:16,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:17,198 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:18,259 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:18,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:18,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:20,406 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:52:21,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:21,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:22,261 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:22,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:23,643 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:23,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:24,261 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:26,262 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:26,652 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:26,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:27,807 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:28,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:28,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:29,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:52:29,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:52:29,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:52:30,263 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:32,124 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:52:32,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:32,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:32,264 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:33,111 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:34,264 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:35,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:35,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:37,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:37,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:38,266 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:38,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:40,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:40,040 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:40,266 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:42,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:42,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:42,267 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:43,716 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:44,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:52:44,410 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:52:44,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:52:45,204 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:45,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:46,268 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:47,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:47,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:48,269 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:49,018 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:50,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:50,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:50,270 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:50,407 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:52:52,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:52,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:52,270 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:54,322 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:52:55,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:55,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:56,272 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:57,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:57,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:58,273 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:52:59,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:59,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:52:59,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:52:59,410 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:52:59,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:52:59,564 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:00,274 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:02,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:02,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:02,302 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:53:02,303 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:53:02,303 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:53:02,304 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:53:03,274 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:53:04,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:04,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:04,275 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:04,933 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:05,275 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:06,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:06,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:06,275 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:08,276 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:09,610 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:53:09,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:09,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:10,241 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:12,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:12,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:12,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:14,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:14,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:53:14,410 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:53:14,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:14,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:14,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:53:15,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:16,279 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:16,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:16,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:19,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:19,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:20,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:20,409 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:53:20,854 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:21,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:21,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:22,282 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:23,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:23,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:24,282 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:26,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:26,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:26,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:28,284 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:28,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:28,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:29,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:53:29,410 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:53:29,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:53:30,285 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:31,462 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:31,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:31,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:32,285 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:33,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:33,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:36,287 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:36,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:36,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:36,772 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:38,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:38,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:38,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:40,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:41,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:41,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:42,077 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:43,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:43,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:44,290 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:44,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:53:44,410 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:53:44,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:53:46,291 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:46,804 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:53:46,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:46,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:47,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:48,292 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:49,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:49,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:50,410 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:53:52,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:52,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:52,293 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:52,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:54,294 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:55,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:55,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:56,295 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:53:57,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:57,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:53:57,990 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:53:59,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:53:59,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:53:59,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:54:00,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:00,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:00,296 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:02,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:02,204 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:02,297 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:03,299 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:04,298 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:05,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:05,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:07,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:07,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:08,299 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:08,631 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:08,641 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 01:54:08,642 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 01:54:08,642 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 01:54:08,643 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 01:54:09,299 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 01:54:10,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:10,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:10,300 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:11,300 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:12,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:12,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:13,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:14,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:54:14,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:54:14,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:54:14,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:14,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:16,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:16,866 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:19,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:19,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:19,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:20,412 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:54:21,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:21,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:22,303 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:24,750 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:54:24,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:24,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:25,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:26,304 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:27,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:27,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:29,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:54:29,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:54:29,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:54:29,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:29,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:30,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:32,306 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:32,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:32,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:34,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:34,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:36,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:37,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:37,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:38,307 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:39,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:39,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:41,894 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:41,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:41,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:43,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:43,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:44,309 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:44,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:54:44,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:54:44,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:54:46,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:46,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:47,619 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:48,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:48,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:50,310 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:50,414 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:54:52,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:52,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:53,416 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:54,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:54,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:54,311 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:54:56,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:56,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:58,662 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:54:58,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:58,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:54:59,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:54:59,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:54:59,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:55:00,314 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:02,112 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:55:02,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:02,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:04,597 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:04,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:04,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:06,315 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:07,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:07,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:09,868 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:10,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:10,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:12,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:12,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:12,316 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:14,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:55:14,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:55:14,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:55:14,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:14,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:15,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:16,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:16,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:18,318 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:19,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:19,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:20,416 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:55:21,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:21,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:21,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:22,319 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:24,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:24,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:26,775 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:26,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:26,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:28,320 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:29,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:55:29,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:55:29,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:55:29,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:29,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:31,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:31,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:32,602 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:34,322 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:34,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:34,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:36,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:36,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:37,936 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:39,786 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:55:39,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:39,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:40,323 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:42,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:42,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:43,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:44,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:55:44,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:55:44,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:44,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:55:44,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:46,328 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:47,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:47,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:48,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:49,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:49,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:50,418 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:55:51,329 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:52,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:52,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:54,414 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:54,420 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:55:54,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:57,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:57,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:57,330 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:55:59,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:59,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:55:59,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:55:59,411 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:55:59,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:55:59,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:02,028 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:02,040 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:03,332 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:04,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:04,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:05,049 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:06,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:06,309 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:09,334 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:09,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:09,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:10,623 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:11,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:11,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:14,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:14,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:14,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:56:14,412 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:56:14,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:56:15,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:15,627 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:17,696 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:56:17,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:17,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:19,336 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:20,419 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:56:20,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:20,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:21,420 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:22,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:22,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:25,338 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:25,771 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:25,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:26,450 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:27,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:27,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:29,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:56:29,412 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:56:29,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:56:30,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:30,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:31,340 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:31,593 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:32,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:32,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:35,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:35,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:36,704 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:37,341 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:37,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:37,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:40,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:40,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:42,327 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:42,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:42,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:43,343 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:44,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:56:44,412 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:56:44,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:56:45,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:45,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:47,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:47,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:47,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:48,018 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:50,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:50,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:50,420 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:56:52,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:52,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:53,346 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:53,589 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:55,701 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:56:55,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:55,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:58,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:58,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:56:59,223 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:56:59,350 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:56:59,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:56:59,412 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:56:59,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:57:00,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:00,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:03,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:03,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:04,821 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:05,352 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:05,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:05,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:08,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:08,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:10,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:10,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:10,451 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:11,354 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:13,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:13,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:14,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:57:14,412 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:57:14,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:57:15,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:15,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:15,355 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:15,578 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:18,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:18,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:20,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:20,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:20,421 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:57:21,356 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:21,423 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:22,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:22,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:25,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:25,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:27,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:27,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:27,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:27,358 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:29,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:57:29,413 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:57:29,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:57:30,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:30,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:32,846 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:33,359 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:33,545 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:57:33,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:33,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:35,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:35,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:38,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:38,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:38,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:39,361 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:40,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:40,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:43,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:43,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:43,362 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:43,977 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:44,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:57:44,413 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:57:44,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:57:45,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:45,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:48,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:48,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:49,364 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:49,652 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:50,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:50,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:50,423 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:57:53,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:53,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:55,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:55,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:55,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:57:55,365 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:57:58,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:58,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:57:59,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:57:59,413 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:57:59,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:58:00,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:00,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:00,576 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:01,367 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:02,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:02,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:04,866 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:04,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:06,435 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:07,369 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:07,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:07,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:11,137 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:58:11,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:11,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:11,370 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:12,041 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:14,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:14,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:14,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:58:14,413 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:58:14,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:58:16,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:16,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:17,371 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:17,689 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:19,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:19,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:20,425 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:58:21,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:21,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:23,373 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:23,446 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:23,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:23,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:25,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:25,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:28,652 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:28,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:29,037 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:29,374 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:29,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:58:29,413 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:58:29,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:58:30,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:30,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:33,541 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:33,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:34,614 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:35,376 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:35,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:35,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:38,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:38,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:40,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:40,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:40,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:41,377 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:43,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:43,280 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:44,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:58:44,413 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:58:44,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:58:45,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:45,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:45,378 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:45,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:48,685 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:58:48,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:48,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:50,427 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:58:51,381 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:51,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:51,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:51,428 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:53,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:53,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:56,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:56,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:57,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:58:57,382 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:58:58,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:58,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:58:59,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:58:59,413 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:58:59,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:59:01,100 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:01,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:02,821 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:03,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:03,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:03,384 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:05,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:05,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:07,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:08,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:08,350 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:09,385 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:10,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:10,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:12,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:12,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:13,387 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:13,949 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:14,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:59:14,413 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:59:14,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:59:15,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:15,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:17,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:17,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:19,388 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:19,567 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:20,428 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:59:20,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:20,767 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:22,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:22,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:25,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:25,390 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:26,013 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 01:59:26,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:26,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:28,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:28,777 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:29,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:59:29,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:59:29,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:59:30,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:30,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:30,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:31,392 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:33,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:33,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:35,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:35,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:36,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:37,393 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:38,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:38,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:40,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:40,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:41,394 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:42,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:43,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:43,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:44,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:59:44,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:59:44,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 01:59:45,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:45,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:47,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:47,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:48,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:48,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:50,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:50,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:50,430 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 01:59:53,214 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:53,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:53,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:53,398 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:55,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:55,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:58,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:58,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 01:59:58,811 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 01:59:59,400 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 01:59:59,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 01:59:59,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 01:59:59,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:00:00,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:00,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:03,628 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:00:03,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:03,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:04,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:05,402 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:06,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:06,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:08,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:08,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:09,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:09,959 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:11,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:11,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:13,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:13,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:14,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:00:14,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:00:14,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:00:15,405 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:15,573 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:16,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:16,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:18,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:18,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:20,431 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:00:21,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:21,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:21,406 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:21,436 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:23,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:23,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:26,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:26,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:26,769 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:27,408 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:28,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:28,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:29,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:00:29,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:00:29,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:00:31,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:31,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:32,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:33,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:33,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:33,410 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:35,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:35,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:37,411 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:37,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:37,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:37,926 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:41,080 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:00:41,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:41,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:43,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:43,524 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:44,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:44,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:44,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:00:44,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:00:44,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:00:46,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:46,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:49,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:49,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:49,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:49,415 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:50,432 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:00:51,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:51,235 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:54,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:54,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:54,762 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:00:55,417 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:00:56,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:56,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:58,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:58,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:00:59,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:00:59,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:00:59,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:01:00,631 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:00,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:00,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:01,418 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:03,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:03,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:05,419 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:05,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:05,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:06,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:08,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:08,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:10,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:10,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:11,421 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:11,825 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:12,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:12,757 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:14,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:01:14,415 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:01:14,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:01:14,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:14,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:17,423 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:17,497 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:18,119 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:01:18,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:18,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:20,434 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:01:20,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:20,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:22,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:22,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:23,035 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:23,424 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:25,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:25,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:27,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:27,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:28,594 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:29,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:01:29,415 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:01:29,426 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:29,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:01:30,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:30,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:32,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:32,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:33,427 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:34,202 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:35,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:35,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:37,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:37,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:39,429 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:39,766 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:40,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:40,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:42,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:42,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:44,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:44,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:44,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:01:44,415 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:01:44,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:01:45,430 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:45,575 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:46,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:47,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:49,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:49,046 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:50,435 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:01:51,432 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:51,437 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:51,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:51,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:55,319 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:01:55,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:55,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:56,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:01:57,434 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:01:58,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:58,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:01:59,415 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:01:59,415 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:01:59,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:02:00,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:00,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:01,435 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:02,229 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:03,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:03,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:05,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:05,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:07,437 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:07,826 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:08,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:08,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:10,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:10,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:13,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:13,234 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:13,418 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:13,438 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:14,415 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:02:14,415 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:02:14,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:02:15,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:15,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:18,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:18,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:19,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:19,440 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:20,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:20,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:20,437 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:02:23,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:23,105 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:24,704 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:25,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:25,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:25,441 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:27,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:27,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:29,415 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:02:29,415 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:02:29,442 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:29,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:02:30,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:30,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:30,600 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:33,381 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:02:33,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:33,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:35,444 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:35,905 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:36,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:36,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:38,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:38,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:41,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:41,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:41,446 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:41,508 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:43,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:43,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:44,415 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:02:44,415 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:02:44,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:02:46,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:46,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:47,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:47,448 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:48,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:48,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:50,439 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:02:50,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:51,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:52,700 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:53,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:53,040 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:53,450 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:55,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:55,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:57,451 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:02:57,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:57,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:02:58,296 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:02:59,415 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:02:59,415 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:02:59,555 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:03:00,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:00,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:02,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:02,755 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:03,453 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:03:03,888 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:05,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:05,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:07,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:07,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:09,440 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:09,454 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:03:10,941 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:03:10,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:10,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:13,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:13,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:14,415 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:03:14,416 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:03:14,556 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:03:14,636 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:15,456 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:03:15,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:15,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:18,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:18,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:20,440 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:03:20,443 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:20,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:20,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:21,458 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:03:23,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:23,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:25,459 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:03:25,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:25,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:26,387 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:28,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:28,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:28,500 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 02:03:28,502 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 02:03:28,502 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 02:03:28,503 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 02:03:29,415 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:03:29,416 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:03:29,465 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 02:03:29,466 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:03:29,560 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:03:30,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:30,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:31,569 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:32,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:33,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:36,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:36,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:37,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:37,467 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:03:38,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:39,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:41,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:42,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:42,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:44,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:44,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:44,710 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:03:44,808 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:03:44,809 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:03:46,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:46,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:47,964 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:50,037 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:03:50,442 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:03:51,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:51,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:53,076 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:53,443 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:54,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:56,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:56,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:58,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:03:58,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:58,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:03:59,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:03:59,678 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:03:59,679 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:04:00,440 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-28 02:04:00,440 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 02:04:00,442 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 02:04:00,442 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 02:04:00,442 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 02:04:00,442 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 02:04:00,443 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 02:04:00,443 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 02:04:00,443 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 02:04:00,443 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 02:04:00,443 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 02:04:00,443 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-28 02:04:00,443 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-28 02:04:00,444 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-28 02:04:00,473 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-28 02:04:00,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:00,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:01,474 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-28 02:04:03,152 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:03,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:03,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:05,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:05,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:07,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:07,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:08,445 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:09,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:09,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:12,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:12,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:13,446 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:14,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:14,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:14,667 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:04:14,678 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:04:14,679 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:04:16,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:16,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:18,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:18,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:18,877 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:20,443 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:04:21,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:21,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:23,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:23,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:24,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:26,394 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:04:26,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:26,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:28,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:28,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:29,445 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:29,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:04:29,679 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:04:29,679 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:04:30,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:30,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:33,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:33,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:34,882 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:35,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:35,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:37,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:37,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:39,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:39,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:39,882 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:42,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:42,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:44,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:44,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:44,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:04:44,679 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:04:44,679 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:04:45,811 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:47,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:47,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:49,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:49,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:50,445 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:04:51,446 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:51,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:51,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:53,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:53,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:55,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:56,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:56,446 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:04:58,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:58,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:04:59,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:04:59,679 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:04:59,679 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:05:01,067 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-28 02:05:01,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:01,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:01,850 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:05:03,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:03,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:05,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:05,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:06,851 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:05:08,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:08,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:10,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:10,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:11,851 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:05:12,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:12,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:14,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-28 02:05:14,679 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-28 02:05:14,679 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-28 02:05:14,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:14,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:17,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:17,419 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:17,849 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:05:19,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:19,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:20,446 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-28 02:05:21,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:21,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:23,448 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-28 02:05:23,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-28 02:05:23,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported