diff --git "a/wandb/run-20240327_125828-wlmqyk6v/logs/debug-internal.log" "b/wandb/run-20240327_125828-wlmqyk6v/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240327_125828-wlmqyk6v/logs/debug-internal.log" @@ -0,0 +1,5978 @@ +2024-03-27 12:58:28,742 INFO StreamThr :1390067 [internal.py:wandb_internal():86] W&B internal server running at pid: 1390067, started at: 2024-03-27 12:58:28.741270 +2024-03-27 12:58:28,744 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status +2024-03-27 12:58:28,746 INFO WriterThread:1390067 [datastore.py:open_for_write():87] open: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/run-wlmqyk6v.wandb +2024-03-27 12:58:28,746 DEBUG SenderThread:1390067 [sender.py:send():382] send: header +2024-03-27 12:58:28,816 DEBUG SenderThread:1390067 [sender.py:send():382] send: run +2024-03-27 12:58:29,131 INFO SenderThread:1390067 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files +2024-03-27 12:58:29,131 INFO SenderThread:1390067 [sender.py:_start_run_threads():1136] run started: wlmqyk6v with start time 1711540708.740468 +2024-03-27 12:58:29,137 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: check_version +2024-03-27 12:58:29,138 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: check_version +2024-03-27 12:58:29,165 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: run_start +2024-03-27 12:58:29,180 DEBUG HandlerThread:1390067 [system_info.py:__init__():27] System info init +2024-03-27 12:58:29,180 DEBUG HandlerThread:1390067 [system_info.py:__init__():42] System info init done +2024-03-27 12:58:29,180 INFO HandlerThread:1390067 [system_monitor.py:start():194] Starting system monitor +2024-03-27 12:58:29,180 INFO SystemMonitor:1390067 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-27 12:58:29,181 INFO HandlerThread:1390067 [system_monitor.py:probe():214] Collecting system info +2024-03-27 12:58:29,182 INFO SystemMonitor:1390067 [interfaces.py:start():190] Started cpu monitoring +2024-03-27 12:58:29,182 INFO SystemMonitor:1390067 [interfaces.py:start():190] Started disk monitoring +2024-03-27 12:58:29,184 INFO SystemMonitor:1390067 [interfaces.py:start():190] Started gpu monitoring +2024-03-27 12:58:29,185 INFO SystemMonitor:1390067 [interfaces.py:start():190] Started memory monitoring +2024-03-27 12:58:29,187 INFO SystemMonitor:1390067 [interfaces.py:start():190] Started network monitoring +2024-03-27 12:58:29,232 DEBUG HandlerThread:1390067 [system_info.py:probe():151] Probing system +2024-03-27 12:58:29,238 DEBUG HandlerThread:1390067 [system_info.py:_probe_git():136] Probing git +2024-03-27 12:58:29,249 DEBUG HandlerThread:1390067 [system_info.py:_probe_git():144] Probing git done +2024-03-27 12:58:29,249 DEBUG HandlerThread:1390067 [system_info.py:probe():199] Probing system done +2024-03-27 12:58:29,249 DEBUG HandlerThread:1390067 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-03-27T11:58:29.232856', 'startedAt': '2024-03-27T11:58:28.736770', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=distil-whisper/distil-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_1', '--dataset_config_name=hi', '--language=hindi', '--train_split_name=train+validation', '--eval_split_name=test', '--max_steps=5000', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=32', '--logging_steps=25', '--learning_rate=1e-4', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--generation_max_length=225', '--preprocessing_num_workers=1', '--dataloader_num_workers=4', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--gradient_checkpointing', '--fp16', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-large-v3-hi-ft', 'commit': '40c686df113c0e98e7363c1bd523f58d11848fc0'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/distil-large-v3-hi-ft', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2170.403765625001, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 3142.167, 'min': 1500.0, 'max': 2250.0}, {'current': 1693.807, 'min': 1500.0, 'max': 2250.0}, {'current': 1693.66, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.174, 'min': 1500.0, 'max': 2250.0}, {'current': 1756.198, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.0, 'min': 1500.0, 'max': 2250.0}, {'current': 1650.564, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.469, 'min': 1500.0, 'max': 2250.0}, {'current': 1782.824, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.775, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.555, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.692, 'min': 1500.0, 'max': 2250.0}, {'current': 2239.537, 'min': 1500.0, 'max': 2250.0}, {'current': 1717.572, 'min': 1500.0, 'max': 2250.0}, {'current': 1740.418, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.353, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.306, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.383, 'min': 1500.0, 'max': 2250.0}, {'current': 1797.94, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.563, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.737, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.785, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.26, 'min': 1500.0, 'max': 2250.0}, {'current': 1790.759, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.862, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.393, 'min': 1500.0, 'max': 2250.0}, {'current': 1778.215, 'min': 1500.0, 'max': 2250.0}, {'current': 1797.531, 'min': 1500.0, 'max': 2250.0}, {'current': 1651.956, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.074, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.083, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.94, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.364, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.524, 'min': 1500.0, 'max': 2250.0}, {'current': 2694.844, 'min': 1500.0, 'max': 2250.0}, {'current': 1696.681, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.48, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.042, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.014, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.604, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.327, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.912, 'min': 1500.0, 'max': 2250.0}, {'current': 1693.123, 'min': 1500.0, 'max': 2250.0}, {'current': 3302.73, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.31, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.051, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.762, 'min': 1500.0, 'max': 2250.0}, {'current': 1659.911, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.156, 'min': 1500.0, 'max': 2250.0}, {'current': 1665.033, 'min': 1500.0, 'max': 2250.0}, {'current': 1730.449, 'min': 1500.0, 'max': 2250.0}, {'current': 1667.591, 'min': 1500.0, 'max': 2250.0}, {'current': 1678.524, 'min': 1500.0, 'max': 2250.0}, {'current': 3380.556, 'min': 1500.0, 'max': 2250.0}, {'current': 1690.177, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.329, 'min': 1500.0, 'max': 2250.0}, {'current': 1689.671, 'min': 1500.0, 'max': 2250.0}, {'current': 1687.925, 'min': 1500.0, 'max': 2250.0}, {'current': 1688.548, 'min': 1500.0, 'max': 2250.0}, {'current': 3388.168, 'min': 1500.0, 'max': 2250.0}, {'current': 1692.888, 'min': 1500.0, 'max': 2250.0}, {'current': 1690.168, 'min': 1500.0, 'max': 2250.0}, {'current': 3388.021, 'min': 1500.0, 'max': 2250.0}, {'current': 1692.056, 'min': 1500.0, 'max': 2250.0}, {'current': 3387.636, 'min': 1500.0, 'max': 2250.0}, {'current': 1689.004, 'min': 1500.0, 'max': 2250.0}, {'current': 1687.968, 'min': 1500.0, 'max': 2250.0}, {'current': 1687.714, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.29, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.495, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.513, 'min': 1500.0, 'max': 2250.0}, {'current': 1452.265, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.895, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.828, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.173, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.185, 'min': 1500.0, 'max': 2250.0}, {'current': 2921.311, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.587, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.206, 'min': 1500.0, 'max': 2250.0}, {'current': 1683.624, 'min': 1500.0, 'max': 2250.0}, {'current': 2446.569, 'min': 1500.0, 'max': 2250.0}, {'current': 2445.849, 'min': 1500.0, 'max': 2250.0}, {'current': 2440.241, 'min': 1500.0, 'max': 2250.0}, {'current': 2465.162, 'min': 1500.0, 'max': 2250.0}, {'current': 2450.373, 'min': 1500.0, 'max': 2250.0}, {'current': 1962.985, 'min': 1500.0, 'max': 2250.0}, {'current': 2409.641, 'min': 1500.0, 'max': 2250.0}, {'current': 2471.639, 'min': 1500.0, 'max': 2250.0}, {'current': 2460.459, 'min': 1500.0, 'max': 2250.0}, {'current': 2464.668, 'min': 1500.0, 'max': 2250.0}, {'current': 2475.763, 'min': 1500.0, 'max': 2250.0}, {'current': 2467.527, 'min': 1500.0, 'max': 2250.0}, {'current': 2201.123, 'min': 1500.0, 'max': 2250.0}, {'current': 1895.971, 'min': 1500.0, 'max': 2250.0}, {'current': 2443.965, 'min': 1500.0, 'max': 2250.0}, {'current': 2476.288, 'min': 1500.0, 'max': 2250.0}, {'current': 3288.307, 'min': 1500.0, 'max': 2250.0}, {'current': 2493.81, 'min': 1500.0, 'max': 2250.0}, {'current': 2565.406, 'min': 1500.0, 'max': 2250.0}, {'current': 2039.03, 'min': 1500.0, 'max': 2250.0}, {'current': 2453.201, 'min': 1500.0, 'max': 2250.0}, {'current': 2474.673, 'min': 1500.0, 'max': 2250.0}, {'current': 2440.26, 'min': 1500.0, 'max': 2250.0}, {'current': 2459.283, 'min': 1500.0, 'max': 2250.0}, {'current': 1695.187, 'min': 1500.0, 'max': 2250.0}, {'current': 1685.3, 'min': 1500.0, 'max': 2250.0}, {'current': 1676.674, 'min': 1500.0, 'max': 2250.0}, {'current': 3348.729, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.959, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.686, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.511, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.74, 'min': 1500.0, 'max': 2250.0}, {'current': 1549.158, 'min': 1500.0, 'max': 2250.0}, {'current': 1835.582, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.186, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.447, 'min': 1500.0, 'max': 2250.0}, {'current': 1693.83, 'min': 1500.0, 'max': 2250.0}, {'current': 3377.374, 'min': 1500.0, 'max': 2250.0}, {'current': 1693.537, 'min': 1500.0, 'max': 2250.0}, {'current': 1739.397, 'min': 1500.0, 'max': 2250.0}, {'current': 1695.165, 'min': 1500.0, 'max': 2250.0}, {'current': 1693.65, 'min': 1500.0, 'max': 2250.0}, {'current': 1695.685, 'min': 1500.0, 'max': 2250.0}, {'current': 3401.224, 'min': 1500.0, 'max': 2250.0}, {'current': 1695.872, 'min': 1500.0, 'max': 2250.0}, {'current': 1695.855, 'min': 1500.0, 'max': 2250.0}, {'current': 3330.83, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.199, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1497.0357131958008}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-03-27 12:58:29,249 INFO HandlerThread:1390067 [system_monitor.py:probe():224] Finished collecting system info +2024-03-27 12:58:29,249 INFO HandlerThread:1390067 [system_monitor.py:probe():227] Publishing system info +2024-03-27 12:58:29,250 INFO HandlerThread:1390067 [system_monitor.py:probe():229] Finished publishing system info +2024-03-27 12:58:29,254 DEBUG SenderThread:1390067 [sender.py:send():382] send: files +2024-03-27 12:58:29,254 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-27 12:58:29,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: python_packages +2024-03-27 12:58:29,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:58:29,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: python_packages +2024-03-27 12:58:29,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:58:29,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:58:29,274 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:29,284 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:29,472 DEBUG SenderThread:1390067 [sender.py:send():382] send: telemetry +2024-03-27 12:58:29,473 DEBUG SenderThread:1390067 [sender.py:send():382] send: config +2024-03-27 12:58:29,473 DEBUG SenderThread:1390067 [sender.py:send():382] send: metric +2024-03-27 12:58:29,473 DEBUG SenderThread:1390067 [sender.py:send():382] send: telemetry +2024-03-27 12:58:29,473 DEBUG SenderThread:1390067 [sender.py:send():382] send: metric +2024-03-27 12:58:29,473 WARNING SenderThread:1390067 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-27 12:58:29,474 DEBUG SenderThread:1390067 [sender.py:send():382] send: telemetry +2024-03-27 12:58:29,768 INFO wandb-upload_0:1390067 [upload_job.py:push():131] Uploaded file /tmp/tmpsts4ntg5wandb/blimuksw-wandb-metadata.json +2024-03-27 12:58:30,133 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/requirements.txt +2024-03-27 12:58:30,133 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:58:30,133 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-metadata.json +2024-03-27 12:58:31,943 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:31,953 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:32,133 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:58:33,972 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:33,983 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:34,475 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:58:36,471 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:36,481 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:38,502 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:38,516 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:39,476 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:58:40,904 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:40,913 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:42,136 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:58:42,935 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:42,949 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:44,137 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:58:44,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:58:44,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:58:44,262 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:58:45,446 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:58:45,529 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:45,542 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:47,564 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:47,575 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:48,138 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:58:50,139 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:58:50,576 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:50,588 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:51,150 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:58:52,608 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:52,618 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:54,140 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:58:55,603 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:55,626 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:56,577 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:58:57,645 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:57,657 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:58:58,142 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:58:59,258 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:58:59,259 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:58:59,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:59:00,142 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:00,570 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:00,582 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:02,028 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:02,143 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:02,602 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:02,612 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:03,143 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/config.yaml +2024-03-27 12:59:06,007 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:59:06,027 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:06,042 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:06,145 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:07,862 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:08,145 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:08,935 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:08,945 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:10,964 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:10,976 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:12,146 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:13,324 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:13,891 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:13,903 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:14,147 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:14,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:59:14,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:59:14,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:59:15,925 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:15,936 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:16,148 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:18,789 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:18,837 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:18,851 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:20,149 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:20,868 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:20,877 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:22,150 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:23,914 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:23,924 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:24,150 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:24,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:25,946 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:25,956 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:28,152 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:28,865 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:28,875 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:29,188 DEBUG SystemMonitor:1390067 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-27 12:59:29,190 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 12:59:29,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:59:29,259 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:59:29,262 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:59:29,460 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:30,153 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:30,897 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:30,908 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:34,032 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:34,043 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:34,154 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:34,690 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:36,064 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:36,075 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:36,156 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:38,156 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:39,234 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:39,245 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:40,153 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:41,266 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:41,278 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:42,158 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:44,158 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:44,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:59:44,259 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:59:44,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:59:44,822 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:59:44,842 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:44,858 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:45,394 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:47,779 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:47,791 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:48,160 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:49,812 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:49,823 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:50,160 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:51,077 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:52,161 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:52,935 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:52,948 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:53,834 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 12:59:53,836 DEBUG SenderThread:1390067 [sender.py:send():382] send: metric +2024-03-27 12:59:53,837 DEBUG SenderThread:1390067 [sender.py:send():382] send: metric +2024-03-27 12:59:53,837 DEBUG SenderThread:1390067 [sender.py:send():382] send: metric +2024-03-27 12:59:53,837 DEBUG SenderThread:1390067 [sender.py:send():382] send: metric +2024-03-27 12:59:53,837 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 12:59:53,837 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 12:59:53,838 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 12:59:54,162 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 12:59:54,968 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:54,978 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:56,163 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:56,538 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:59:58,027 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:58,037 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:59:58,163 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 12:59:59,192 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 12:59:59,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:59:59,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:59:59,262 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:00:00,067 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:00,086 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:00,164 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:02,001 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:03,082 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:03,093 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:04,165 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:05,112 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:05,122 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:06,166 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:07,466 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:08,167 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/config.yaml +2024-03-27 13:00:08,168 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:08,169 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:08,179 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:10,204 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:10,215 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:12,169 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:12,924 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:13,105 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:13,115 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:14,170 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:14,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:00:14,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:00:14,262 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:00:15,138 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:15,148 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:18,171 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:18,171 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:18,186 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:18,388 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:20,172 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:20,205 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:20,219 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:22,172 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:23,745 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:00:23,767 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:23,778 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:23,847 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:26,174 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:26,901 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:26,913 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:28,174 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:28,939 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:28,949 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:29,194 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:00:29,195 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:29,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:00:29,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:00:29,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:00:30,175 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:31,977 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:31,987 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:34,008 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:34,021 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:34,176 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:34,782 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:36,177 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:37,086 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:37,104 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:38,177 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:39,125 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:39,135 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:40,248 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:42,067 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:42,083 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:42,178 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:44,107 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:44,116 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:44,180 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:44,259 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:00:44,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:00:44,262 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:00:45,415 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:47,105 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:47,119 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:48,181 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:49,137 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:49,146 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:50,182 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:50,452 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:52,183 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:52,203 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:52,216 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:54,236 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:54,251 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:55,914 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:00:56,185 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:57,181 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:57,198 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:58,186 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:00:59,195 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:00:59,220 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:59,231 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:00:59,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:00:59,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:00:59,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:01:00,186 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:01,460 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:02,131 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:01:02,132 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:01:02,133 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:01:02,133 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:01:02,188 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:01:02,864 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:01:02,885 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:02,901 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:04,189 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:05,682 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:05,696 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:06,189 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:06,835 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:07,715 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:07,726 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:08,189 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:09,746 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:09,763 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:12,190 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:12,298 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:12,529 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:12,539 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:14,192 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:14,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:01:14,260 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:01:14,262 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:01:14,557 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:14,566 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:17,613 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:17,625 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:17,763 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:18,193 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:19,644 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:19,654 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:20,193 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:22,194 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:22,668 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:22,686 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:23,222 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:24,708 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:24,719 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:26,195 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:27,751 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:27,761 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:28,196 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:28,682 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:29,197 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:01:29,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:01:29,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:01:29,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:01:29,787 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:29,796 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:30,199 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:32,859 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:32,870 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:34,143 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:34,201 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:34,894 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:34,908 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:36,201 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:37,824 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:37,834 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:38,202 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:39,613 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:41,449 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:01:41,469 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:41,478 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:42,203 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:43,504 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:43,515 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:44,204 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:44,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:01:44,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:01:44,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:01:45,417 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:46,253 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:46,266 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:48,205 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:48,287 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:48,298 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:50,206 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:50,550 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:51,215 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:51,226 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:52,206 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:53,245 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:53,257 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:56,018 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:01:56,207 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:56,292 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:56,302 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:58,208 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:01:58,321 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:58,339 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:01:59,199 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:01:59,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:01:59,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:01:59,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:02:00,208 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:01,357 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:01,372 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:01,480 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:03,393 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:03,403 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:04,209 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:06,210 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:06,310 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:06,319 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:06,941 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:08,210 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:08,339 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:08,349 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:10,430 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:02:10,432 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:02:10,432 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:02:10,432 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:02:11,211 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:02:11,382 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:11,395 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:12,211 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:12,433 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:13,414 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:13,424 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:14,212 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:14,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:02:14,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:02:14,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:02:16,443 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:16,456 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:17,863 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:18,213 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:19,993 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:02:20,012 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:20,022 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:20,214 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:22,214 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:23,025 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:23,036 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:23,326 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:25,056 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:25,066 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:26,215 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:27,969 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:27,984 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:28,216 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:28,787 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:29,201 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:02:29,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:02:29,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:02:29,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:02:30,004 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:30,014 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:30,217 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:32,036 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:32,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:34,218 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:34,250 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:34,909 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:34,920 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:36,219 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:36,942 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:36,952 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:38,219 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:39,711 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:40,009 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:40,021 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:42,050 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:42,061 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:42,221 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:44,221 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:44,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:02:44,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:02:44,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:02:45,194 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:45,204 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:45,426 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:47,223 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:47,239 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:48,222 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:50,222 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:50,277 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:50,296 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:50,644 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:52,223 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:52,318 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:52,329 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:54,349 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:54,358 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:56,111 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:02:56,224 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:57,669 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:02:57,691 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:57,708 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:02:58,225 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:02:59,203 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:02:59,260 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:02:59,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:02:59,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:03:00,225 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:00,718 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:00,727 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:01,582 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:02,751 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:02,760 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:04,226 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:05,805 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:05,817 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:06,227 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:07,048 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:07,835 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:07,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:08,227 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:10,988 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:10,998 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:12,229 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:12,509 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:13,020 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:13,031 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:14,229 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:14,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:03:14,262 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:03:14,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:03:15,982 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:15,993 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:17,971 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:18,014 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:18,025 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:18,230 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:18,730 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:03:18,731 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:03:18,731 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:03:18,732 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:03:19,230 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:03:20,230 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:21,109 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:21,124 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:22,231 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:23,144 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:23,157 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:23,435 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:26,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:26,056 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:26,232 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:28,078 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:28,091 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:28,232 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:28,894 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:29,204 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:03:29,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:03:29,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:03:29,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:03:30,233 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:31,144 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:31,156 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:33,173 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:33,183 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:34,234 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:34,357 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:36,235 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:36,742 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:03:36,765 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:36,776 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:38,236 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:39,551 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:39,565 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:39,817 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:41,584 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:41,598 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:42,237 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:43,620 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:43,630 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:44,237 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:44,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:03:44,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:03:44,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:03:45,449 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:46,390 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:46,401 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:48,238 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:48,427 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:48,437 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:50,238 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:50,741 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:51,446 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:51,462 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:52,239 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:53,487 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:53,497 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:56,208 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:03:56,240 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:56,424 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:56,434 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:58,241 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:03:58,454 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:58,468 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:03:59,206 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:03:59,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:03:59,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:03:59,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:04:00,242 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:01,490 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:01,509 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:01,673 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:03,528 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:03,538 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:04,243 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:06,243 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:06,471 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:06,492 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:07,142 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:08,244 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:08,512 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:08,522 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:11,638 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:11,651 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:12,245 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:12,606 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:14,245 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:14,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:04:14,262 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:04:14,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:04:15,224 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:04:15,246 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:15,256 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:18,067 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:18,246 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:18,382 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:18,394 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:20,247 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:20,413 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:20,423 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:22,247 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:22,446 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:22,458 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:23,528 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:25,257 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:25,268 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:26,249 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:27,014 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:04:27,015 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:04:27,016 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:04:27,016 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:04:27,249 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:04:27,286 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:27,301 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:28,250 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:29,017 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:29,207 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:04:29,261 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:04:29,261 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:04:29,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:04:29,323 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:29,337 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:30,250 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:32,256 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:32,267 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:34,251 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:34,287 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:34,299 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:34,449 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:36,252 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:37,360 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:37,374 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:38,252 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:39,395 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:39,404 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:39,909 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:42,253 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:42,455 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:42,469 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:44,255 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:44,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:04:44,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:04:44,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:04:44,488 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:44,498 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:45,489 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:47,402 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:47,412 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:48,257 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:49,432 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:49,444 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:50,257 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:50,830 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:52,258 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:53,008 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:04:53,028 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:53,037 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:56,043 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:56,053 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:56,259 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:56,293 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:04:58,074 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:58,090 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:04:58,259 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:04:59,209 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:04:59,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:04:59,264 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:04:59,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:05:00,108 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:00,124 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:00,260 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:01,758 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:03,152 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:03,167 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:04,261 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:05,186 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:05,198 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:06,262 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:07,223 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:08,117 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:08,127 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:08,262 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:10,150 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:10,159 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:12,263 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:12,687 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:13,190 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:13,201 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:14,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:05:14,264 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:05:14,264 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:14,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:05:15,230 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:15,240 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:18,114 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:18,128 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:18,153 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:18,265 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:20,150 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:20,160 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:20,266 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:22,266 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:23,397 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:23,411 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:23,617 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:25,433 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:25,442 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:26,267 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:28,268 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:28,423 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:28,433 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:29,078 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:29,210 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:05:29,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:05:29,264 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:05:29,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:05:30,269 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:32,017 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:05:32,035 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:32,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:34,269 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:34,538 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:35,026 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:35,038 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:35,296 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:05:35,297 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:05:35,298 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:05:35,298 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:05:36,270 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:05:36,270 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:37,062 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:37,071 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:38,270 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:40,001 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:40,084 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:40,095 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:42,115 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:42,125 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:42,272 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:44,144 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:44,157 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:44,263 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:05:44,264 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:05:44,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:05:44,273 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:45,431 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:46,914 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:46,928 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:48,273 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:48,951 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:48,961 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:50,274 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:50,920 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:51,001 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:51,016 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:52,274 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:53,765 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:53,778 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:55,805 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:55,816 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:56,275 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:56,382 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:05:57,837 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:57,849 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:05:58,276 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:05:59,212 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:05:59,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:05:59,264 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:05:59,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:06:00,277 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:00,644 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:00,665 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:01,840 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:02,683 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:02,692 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:04,278 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:05,691 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:05,705 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:06,279 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:07,305 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:08,280 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:09,273 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:06:09,292 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:09,307 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:11,326 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:11,338 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:12,281 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:12,773 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:14,085 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:14,100 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:14,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:06:14,264 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:06:14,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:06:14,282 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:16,121 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:16,134 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:18,242 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:18,284 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:19,185 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:19,200 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:20,285 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:21,221 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:21,232 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:22,286 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:23,707 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:24,340 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:24,352 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:26,287 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:26,373 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:26,388 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:28,287 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:29,169 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:29,214 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:06:29,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:06:29,265 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:06:29,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:06:29,306 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:29,322 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:30,288 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:31,342 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:31,351 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:34,266 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:34,278 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:34,290 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:34,630 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:36,290 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:36,299 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:36,310 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:38,291 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:38,336 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:38,349 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:40,111 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:41,073 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:41,083 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:42,292 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:43,101 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:43,113 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:43,596 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:06:43,597 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:06:43,598 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:06:43,598 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:06:44,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:06:44,264 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:06:44,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:06:44,293 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:06:44,293 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:45,293 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:45,409 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:46,610 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:06:46,628 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:46,643 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:48,294 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:49,708 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:49,719 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:50,295 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:51,032 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:51,739 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:51,754 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:52,295 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:54,915 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:54,930 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:56,296 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:56,491 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:06:56,951 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:56,970 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:58,297 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:06:59,216 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:06:59,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:06:59,264 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:06:59,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:06:59,881 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:06:59,894 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:00,297 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:01,921 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:01,930 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:01,954 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:04,299 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:05,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:05,068 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:06,300 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:07,090 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:07,105 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:07,415 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:08,300 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:10,025 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:10,039 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:12,059 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:12,069 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:12,301 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:12,899 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:14,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:07:14,265 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:07:14,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:07:14,302 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:15,098 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:15,109 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:17,128 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:17,142 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:18,303 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:18,386 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:20,261 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:20,272 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:20,304 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:22,296 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:22,304 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:22,314 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:23,850 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:25,857 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:07:25,876 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:25,886 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:26,306 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:28,306 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:28,744 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:28,756 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:29,217 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:07:29,218 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:29,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:07:29,265 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:07:29,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:07:30,307 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:30,774 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:30,784 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:32,808 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:32,829 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:34,308 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:34,779 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:35,593 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:35,603 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:36,309 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:37,622 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:37,633 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:40,262 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:40,662 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:40,678 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:41,310 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:42,699 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:42,708 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:43,311 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:44,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:07:44,265 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:07:44,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:07:45,311 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:45,418 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:45,767 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:45,780 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:47,800 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:47,811 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:49,313 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:50,711 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:50,726 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:51,206 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:51,316 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:51,962 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:07:51,963 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:07:51,964 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:07:51,964 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:07:52,316 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:07:52,745 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:52,755 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:53,316 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:54,777 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:54,786 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:56,686 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:07:57,318 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:57,697 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:57,713 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:59,219 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:07:59,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:07:59,265 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:07:59,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:07:59,318 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:07:59,732 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:07:59,743 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:01,319 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:02,150 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:03,267 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:08:03,291 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:03,304 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:05,320 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:06,181 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:06,193 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:07,321 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:07,610 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:08,212 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:08,222 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:11,322 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:11,356 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:11,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:13,070 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:13,322 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:13,384 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:13,395 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:14,265 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:08:14,265 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:08:14,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:08:15,323 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:16,418 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:16,431 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:18,452 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:18,462 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:18,531 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:19,324 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:21,325 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:21,417 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:21,427 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:23,326 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:23,451 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:23,461 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:23,995 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:26,488 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:26,499 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:27,327 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:28,518 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:28,529 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:29,220 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:08:29,221 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:29,265 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:08:29,265 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:08:29,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:08:29,327 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:31,328 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:31,720 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:31,731 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:33,751 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:33,775 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:34,929 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:35,330 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:36,956 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:36,968 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:37,330 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:38,986 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:38,995 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:40,396 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:41,331 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:42,442 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:08:42,464 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:42,479 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:43,332 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:44,265 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:08:44,265 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:08:44,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:08:45,242 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:45,253 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:45,333 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:45,456 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:47,272 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:47,283 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:49,334 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:50,260 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:50,270 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:51,334 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:51,339 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:52,286 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:52,297 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:53,335 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:08:55,026 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:55,036 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:57,029 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:08:57,062 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:57,072 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:59,222 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:08:59,265 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:08:59,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:08:59,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:08:59,683 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:08:59,692 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:01,712 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:01,722 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:02,434 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:04,372 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:04,382 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:06,403 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:06,413 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:08,354 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:09,340 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:09,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:09,378 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:11,341 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:11,402 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:11,415 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:11,843 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:09:11,845 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:09:11,846 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:09:11,846 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:09:12,341 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:09:13,341 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:13,847 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:14,265 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:09:14,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:09:14,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:09:14,419 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:14,430 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:16,452 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:16,463 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:17,342 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:19,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:19,343 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:20,041 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:09:20,062 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:20,075 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:21,344 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:23,017 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:23,027 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:24,743 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:25,047 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:25,065 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:25,345 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:27,346 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:28,233 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:28,248 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:29,224 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:09:29,265 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:09:29,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:09:29,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:09:29,346 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:30,268 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:30,280 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:30,392 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:33,348 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:33,353 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:33,365 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:35,348 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:35,387 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:35,397 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:35,678 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:38,328 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:38,338 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:39,349 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:40,356 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:40,369 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:41,145 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:41,350 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:43,351 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:43,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:43,379 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:44,265 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:09:44,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:09:44,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:09:45,398 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:45,410 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:46,611 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:47,352 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:48,574 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:48,589 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:49,352 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:50,609 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:50,619 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:51,353 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:52,075 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:52,645 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:52,660 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:55,354 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:55,411 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:55,421 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:57,354 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:09:57,542 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:09:59,122 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:09:59,141 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:59,153 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:09:59,225 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:09:59,265 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:09:59,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:09:59,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:09:59,355 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:01,178 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:01,192 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:03,005 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:03,357 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:03,952 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:03,970 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:05,358 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:05,991 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:06,001 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:08,468 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:09,137 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:09,151 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:09,359 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:11,172 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:11,187 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:11,360 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:13,262 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:13,282 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:13,360 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:13,932 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:14,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:10:14,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:10:14,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:10:16,392 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:16,402 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:17,362 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:18,421 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:18,437 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:19,362 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:19,518 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:20,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:10:20,275 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:10:20,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:10:20,276 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:10:20,363 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:10:21,347 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:21,362 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:21,363 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:23,381 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:23,394 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:24,977 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:25,365 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:26,427 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:26,454 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:27,366 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:28,482 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:28,498 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:29,227 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:10:29,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:10:29,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:10:29,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:10:29,366 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:30,484 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:31,390 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:31,409 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:33,368 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:33,430 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:33,448 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:35,368 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:35,905 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:36,927 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:10:36,946 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:36,956 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:39,703 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:39,713 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:41,370 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:41,693 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:41,734 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:41,744 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:44,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:10:44,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:10:44,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:10:44,744 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:44,756 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:46,767 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:46,779 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:46,790 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:47,372 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:49,789 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:49,799 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:51,821 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:51,834 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:52,766 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:53,374 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:10:53,862 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:53,872 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:56,637 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:56,662 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:57,802 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:10:58,682 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:58,696 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:10:59,229 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:10:59,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:10:59,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:10:59,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:10:59,376 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:01,649 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:01,659 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:02,847 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:03,679 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:03,689 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:05,378 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:06,650 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:06,663 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:07,848 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:08,684 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:08,694 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:11,379 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:11,696 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:11,707 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:12,856 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:14,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:11:14,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:11:14,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:11:15,379 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:11:15,398 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:15,408 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:17,381 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:17,429 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:17,440 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:17,905 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:20,327 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:20,342 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:22,382 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:22,394 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:22,964 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:23,382 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:25,358 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:25,377 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:27,398 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:27,414 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:28,012 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:29,230 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:11:29,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:11:29,266 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:11:29,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:11:29,384 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:30,230 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:30,249 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:32,278 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:32,294 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:33,052 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:35,293 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:35,314 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:35,386 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:37,336 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:37,347 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:38,053 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:39,368 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:39,382 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:41,388 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:42,118 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:42,132 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:43,101 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:44,161 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:44,174 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:44,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:11:44,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:11:44,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:11:46,197 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:46,208 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:47,389 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:48,396 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:49,027 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:49,049 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:52,603 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:11:52,625 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:52,635 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:53,391 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:53,441 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:54,665 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:55,488 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:57,510 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:57,522 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:58,488 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:11:59,232 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:11:59,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:11:59,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:11:59,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:11:59,394 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:11:59,540 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:11:59,554 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:02,293 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:02,303 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:03,549 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:04,321 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:04,331 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:05,396 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:07,356 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:07,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:08,549 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:09,394 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:09,405 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:11,398 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:12,493 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:12,508 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:13,595 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:14,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:12:14,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:12:14,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:12:14,527 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:14,538 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:17,399 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:17,588 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:17,601 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:18,651 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:19,623 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:19,635 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:21,657 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:21,668 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:23,401 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:23,702 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:24,524 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:24,539 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:26,582 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:26,591 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:28,739 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:29,233 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:12:29,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:12:29,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:12:29,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:12:29,402 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:30,020 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:12:30,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:30,060 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:32,870 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:32,881 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:33,793 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:33,864 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:12:33,865 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:12:33,865 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:12:33,866 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:12:34,404 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:12:34,907 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:34,916 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:35,404 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:38,075 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:38,088 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:38,868 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:40,107 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:40,116 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:41,406 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:43,130 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:43,140 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:44,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:12:44,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:12:44,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:12:44,464 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:45,161 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:45,173 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:47,409 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:48,031 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:48,043 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:49,903 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:50,062 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:50,071 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:52,093 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:52,106 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:53,411 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:54,835 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:54,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:54,950 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:12:56,864 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:56,874 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:59,235 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:12:59,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:12:59,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:12:59,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:12:59,412 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:12:59,763 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:12:59,776 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:00,392 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:01,795 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:01,805 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:04,837 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:04,849 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:05,414 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:06,060 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:08,546 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:13:08,569 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:08,581 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:11,121 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:11,416 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:11,521 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:11,531 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:13,555 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:13,565 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:14,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:13:14,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:13:14,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:13:16,176 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:16,593 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:16,602 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:17,418 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:18,624 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:18,634 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:21,176 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:21,496 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:21,507 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:23,419 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:23,525 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:23,539 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:26,215 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:26,513 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:26,522 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:28,542 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:28,553 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:29,236 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:13:29,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:13:29,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:13:29,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:13:29,422 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:31,443 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:31,537 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:31,549 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:33,568 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:33,578 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:35,423 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:36,463 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:36,478 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:37,326 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:38,502 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:38,514 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:41,425 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:41,499 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:41,524 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:42,382 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:43,547 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:43,564 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:44,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:13:44,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:13:44,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:13:47,054 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:13:47,074 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:47,084 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:47,428 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:47,433 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:49,795 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:49,809 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:51,828 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:51,838 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:52,434 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:53,430 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:13:53,860 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:53,874 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:56,609 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:56,628 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:57,474 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:13:58,646 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:58,667 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:13:59,238 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:13:59,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:13:59,267 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:13:59,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:13:59,432 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:01,644 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:01,666 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:02,489 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:03,684 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:03,707 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:05,435 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:06,831 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:06,857 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:07,565 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:08,903 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:08,912 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:11,437 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:11,757 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:11,767 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:12,610 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:13,786 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:13,798 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:14,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:14:14,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:14:14,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:14:16,715 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:16,724 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:17,439 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:17,643 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:18,743 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:18,752 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:21,755 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:21,769 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:22,645 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:23,441 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:25,425 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:14:25,446 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:25,456 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:27,691 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:28,451 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:28,461 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:29,240 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:14:29,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:14:29,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:14:29,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:14:29,443 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:30,482 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:30,491 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:33,445 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:33,491 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:33,502 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:35,523 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:35,532 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:37,446 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:38,437 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:38,450 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:38,785 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:40,469 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:40,479 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:43,448 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:43,470 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:43,481 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:43,833 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:44,267 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:14:44,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:14:44,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:14:45,502 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:45,513 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:48,501 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:48,514 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:48,875 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:49,450 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:50,544 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:50,555 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:53,554 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:53,573 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:53,917 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:55,453 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:14:55,594 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:55,608 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:58,589 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:58,611 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:14:58,918 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:14:59,241 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:14:59,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:14:59,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:14:59,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:15:00,663 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:00,673 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:01,455 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:04,125 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:15:04,146 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:04,158 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:04,499 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:05,104 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:15:05,105 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:15:05,106 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:15:05,106 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:15:05,456 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:15:06,870 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:06,881 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:07,457 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:08,900 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:08,910 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:10,107 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:12,023 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:12,035 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:13,460 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:14,052 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:14,064 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:14,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:15:14,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:15:14,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:15:15,489 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:17,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:17,073 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:19,092 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:19,102 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:19,462 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:21,146 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:22,074 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:22,089 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:24,107 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:24,117 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:25,464 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:26,189 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:27,185 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:27,200 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:29,219 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:29,233 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:29,243 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:15:29,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:15:29,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:15:29,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:15:31,437 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:31,467 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:32,138 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:32,166 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:34,189 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:34,198 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:37,047 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:37,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:37,299 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:37,469 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:39,097 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:39,116 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:42,339 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:42,692 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:15:42,712 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:42,725 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:43,471 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:44,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:15:44,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:15:44,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:15:45,468 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:45,483 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:47,393 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:47,502 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:47,515 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:49,473 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:50,501 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:50,518 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:52,394 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:52,538 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:52,550 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:55,475 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:15:55,541 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:55,552 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:57,449 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:15:57,573 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:57,584 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:59,245 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:15:59,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:15:59,268 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:15:59,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:15:59,606 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:15:59,619 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:01,477 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:02,335 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:02,346 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:02,488 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:04,368 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:04,378 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:07,390 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:07,411 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:07,480 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:07,540 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:09,434 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:09,444 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:12,440 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:12,455 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:12,596 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:13,482 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:14,268 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:16:14,269 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:16:14,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:16:14,479 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:14,492 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:16,524 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:16,534 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:17,642 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:19,485 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:19,758 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:16:19,777 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:19,789 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:22,643 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:22,655 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:22,665 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:24,685 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:24,697 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:25,487 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:27,694 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:27,818 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:27,837 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:29,247 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:16:29,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:16:29,269 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:16:29,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:16:29,857 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:29,867 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:31,489 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:32,737 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:32,772 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:32,799 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:34,827 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:34,851 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:37,492 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:37,782 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:37,919 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:37,948 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:39,967 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:39,985 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:42,820 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:42,833 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:43,475 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:43,494 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:44,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:16:44,269 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:16:44,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:16:44,852 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:44,863 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:45,494 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:47,909 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:47,919 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:49,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:49,496 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:49,941 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:49,957 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:51,496 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:52,846 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:52,856 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:53,497 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:16:54,738 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:16:54,874 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:54,890 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:58,489 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:16:58,511 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:58,522 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:16:59,249 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:16:59,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:16:59,269 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:16:59,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:16:59,498 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:00,410 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:01,348 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:01,368 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:03,390 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:03,409 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:05,429 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:05,448 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:05,501 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:06,026 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:08,351 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:08,385 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:10,405 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:10,424 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:11,098 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:11,502 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:13,233 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:13,256 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:14,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:17:14,269 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:17:14,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:17:15,281 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:15,301 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:16,136 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:17,333 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:17,356 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:17,504 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:20,051 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:20,071 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:21,137 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:21,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:17:21,277 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:17:21,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:17:21,278 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:17:21,506 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:17:22,091 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:22,104 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:23,506 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:24,129 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:24,963 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:26,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:26,981 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:26,992 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:29,012 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:29,026 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:29,251 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:17:29,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:17:29,270 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:17:29,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:17:29,509 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:31,439 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:31,805 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:31,829 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:35,427 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:17:35,446 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:35,457 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:35,511 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:37,333 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:38,344 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:38,356 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:40,374 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:40,383 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:41,513 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:42,395 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:42,416 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:42,438 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:44,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:17:44,270 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:17:44,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:17:45,250 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:45,260 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:47,281 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:47,292 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:47,460 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:47,515 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:50,145 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:50,160 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:52,182 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:52,193 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:52,520 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:53,517 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:17:55,054 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:55,065 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:57,087 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:57,098 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:57,520 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:17:59,116 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:59,125 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:17:59,252 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:17:59,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:17:59,270 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:17:59,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:17:59,519 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:02,136 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:02,147 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:03,419 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:04,167 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:04,177 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:05,521 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:06,198 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:06,215 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:08,632 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:08,928 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:08,946 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:11,523 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:12,483 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:18:12,502 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:12,512 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:13,688 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:14,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:18:14,270 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:18:14,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:18:14,536 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:14,548 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:17,355 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:17,365 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:17,525 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:18,742 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:19,384 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:19,398 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:22,638 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:22,648 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:23,526 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:23,816 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:24,668 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:24,679 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:27,640 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:27,655 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:28,852 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:29,254 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:18:29,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:18:29,270 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:18:29,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:18:29,528 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:29,676 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:29,685 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:32,771 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:32,783 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:33,901 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:34,801 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:34,811 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:35,530 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:37,870 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:37,884 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:38,902 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:39,905 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:39,917 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:41,532 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:43,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:43,075 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:43,954 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:44,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:18:44,270 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:18:44,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:18:45,095 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:45,105 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:47,535 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:47,974 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:47,984 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:49,010 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:51,878 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:18:51,897 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:51,907 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:53,537 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:54,060 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:54,955 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:54,966 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:56,985 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:56,997 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:59,129 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:18:59,256 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:18:59,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:18:59,270 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:18:59,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:18:59,539 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:18:59,857 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:18:59,867 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:01,887 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:01,899 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:04,167 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:04,900 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:04,910 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:05,541 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:06,932 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:06,941 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:08,963 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:08,974 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:09,168 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:11,543 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:11,717 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:11,729 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:13,773 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:13,783 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:14,224 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:14,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:19:14,270 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:19:14,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:19:16,833 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:16,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:17,545 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:18,864 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:18,879 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:19,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:21,796 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:21,806 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:23,546 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:23,823 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:23,834 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:24,345 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:25,854 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:25,864 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:29,151 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:19:29,173 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:29,186 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:29,258 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:19:29,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:19:29,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:19:29,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:19:29,398 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:29,549 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:32,169 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:32,182 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:34,221 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:34,231 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:34,398 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:35,550 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:37,111 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:37,132 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:39,155 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:39,173 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:39,478 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:41,553 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:42,142 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:42,164 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:44,190 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:44,209 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:44,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:19:44,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:19:44,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:19:45,422 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:47,247 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:47,269 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:47,555 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:49,291 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:49,309 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:50,586 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:52,152 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:52,172 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:52,715 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:19:52,717 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:19:52,717 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:19:52,718 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:19:53,556 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:19:53,557 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:19:54,192 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:54,203 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:55,719 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:19:57,174 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:57,185 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:59,204 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:59,214 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:19:59,260 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:19:59,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:19:59,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:19:59,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:19:59,558 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:01,432 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:02,323 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:02,333 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:04,355 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:04,365 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:05,560 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:06,747 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:07,784 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:20:07,809 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:07,821 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:10,833 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:10,862 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:11,563 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:11,818 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:12,903 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:12,912 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:14,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:20:14,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:20:14,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:20:15,723 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:15,743 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:16,868 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:17,565 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:17,767 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:17,781 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:20,739 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:20,756 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:21,868 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:22,776 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:22,791 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:23,566 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:25,873 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:25,894 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:26,939 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:27,918 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:27,934 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:29,261 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:20:29,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:20:29,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:20:29,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:20:29,568 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:30,795 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:30,814 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:32,497 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:32,836 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:32,848 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:34,873 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:35,096 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:35,571 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:37,603 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:37,621 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:38,042 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:39,639 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:39,652 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:41,574 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:42,643 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:42,664 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:43,116 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:44,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:20:44,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:20:44,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:20:46,446 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:20:46,466 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:46,475 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:47,575 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:48,169 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:49,524 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:49,549 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:51,573 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:51,588 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:53,170 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:53,577 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:20:54,424 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:54,446 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:56,468 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:56,482 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:58,222 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:20:59,262 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:20:59,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:20:59,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:20:59,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:20:59,468 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:20:59,487 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:01,521 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:01,530 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:01,579 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:03,447 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:04,360 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:04,389 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:06,410 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:06,423 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:07,582 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:09,340 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:09,489 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:09,509 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:11,528 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:11,538 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:13,583 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:14,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:21:14,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:21:14,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:21:14,408 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:14,418 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:14,468 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:16,439 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:16,448 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:19,302 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:19,312 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:19,585 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:20,454 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:21,338 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:21,348 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:24,878 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:21:24,897 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:24,907 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:25,507 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:25,587 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:27,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:27,855 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:29,264 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:21:29,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:21:29,271 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:21:29,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:21:29,872 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:29,882 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:30,558 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:31,589 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:32,753 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:32,766 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:34,790 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:34,804 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:35,558 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:37,591 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:37,768 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:37,780 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:39,799 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:39,809 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:40,615 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:41,831 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:41,842 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:43,593 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:44,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:21:44,272 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:21:44,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:21:44,593 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:44,608 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:46,488 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:46,630 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:46,640 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:49,595 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:49,717 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:49,727 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:51,714 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:51,746 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:51,760 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:54,701 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:54,724 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:55,597 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:21:56,743 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:56,761 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:56,773 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:21:59,265 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:21:59,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:21:59,272 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:21:59,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:21:59,638 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:21:59,659 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:01,598 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:01,818 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:03,314 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:22:03,340 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:03,349 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:06,324 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:06,337 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:06,864 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:07,600 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:08,359 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:08,372 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:11,368 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:11,390 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:11,865 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:13,414 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:13,427 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:13,602 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:14,271 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:22:14,272 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:22:14,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:22:16,394 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:16,405 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:17,433 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:18,428 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:18,437 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:19,605 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:21,509 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:21,519 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:22,985 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:23,538 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:23,551 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:24,098 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:22:24,100 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:22:24,100 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:22:24,100 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:22:24,606 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:22:25,607 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:26,463 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:26,475 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:28,101 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:28,493 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:28,503 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:29,267 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:22:29,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:22:29,272 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:22:29,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:22:31,341 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:31,351 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:31,609 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:33,371 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:33,384 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:34,079 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:36,411 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:36,422 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:37,610 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:38,442 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:38,453 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:39,121 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:41,842 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:22:41,866 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:41,880 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:43,612 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:44,175 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:44,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:22:44,272 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:22:44,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:22:44,759 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:44,775 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:46,795 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:46,807 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:49,235 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:49,614 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:49,828 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:49,842 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:51,865 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:51,875 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:54,237 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:54,782 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:54,802 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:55,617 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:22:56,824 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:56,839 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:59,269 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:22:59,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:22:59,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:22:59,272 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:22:59,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:22:59,838 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:22:59,850 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:01,618 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:01,869 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:01,879 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:04,765 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:04,776 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:04,791 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:05,620 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:06,793 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:06,802 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:07,621 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:09,621 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:09,773 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:09,783 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:10,257 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:11,803 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:11,816 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:13,622 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:14,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:23:14,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:23:14,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:23:14,944 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:14,960 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:15,461 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:16,978 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:16,987 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:17,624 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:20,527 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:23:20,551 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:20,563 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:21,352 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:23,479 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:23,490 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:23,626 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:25,511 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:25,523 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:26,888 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:28,550 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:28,571 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:29,270 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:23:29,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:23:29,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:23:29,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:23:29,627 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:30,590 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:30,603 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:32,463 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:33,454 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:33,479 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:35,498 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:35,514 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:35,629 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:37,982 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:38,482 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:38,501 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:40,522 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:40,534 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:41,631 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:43,044 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:43,509 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:43,524 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:44,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:23:44,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:23:44,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:23:45,544 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:45,554 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:47,632 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:48,101 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:48,533 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:48,546 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:50,564 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:50,577 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:53,141 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:53,565 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:53,578 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:53,634 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:23:55,600 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:55,609 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:58,142 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:23:59,158 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:23:59,185 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:59,201 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:23:59,271 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:23:59,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:23:59,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:23:59,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:23:59,637 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:02,027 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:02,038 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:03,446 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:04,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:04,070 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:05,638 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:07,115 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:07,126 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:08,473 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:09,144 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:09,153 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:11,640 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:12,295 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:12,327 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:13,514 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:14,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:24:14,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:24:14,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:24:14,351 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:14,361 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:17,379 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:17,395 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:17,642 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:18,554 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:19,419 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:19,428 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:22,548 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:22,560 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:23,622 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:23,645 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:24,579 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:24,596 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:27,653 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:27,670 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:28,622 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:29,272 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:24:29,273 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:24:29,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:24:29,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:24:29,646 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:29,689 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:29,702 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:31,734 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:31,750 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:34,422 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:34,465 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:34,488 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:35,648 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:38,051 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:24:38,076 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:38,089 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:39,730 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:41,111 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:24:41,112 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:24:41,112 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:24:41,113 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:24:41,250 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:41,260 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:41,650 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:24:41,651 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:43,278 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:43,291 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:44,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:24:44,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:24:44,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:24:45,495 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:46,294 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:46,308 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:47,652 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:48,328 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:48,347 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:50,368 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:50,382 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:51,098 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:53,142 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:53,153 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:53,654 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:24:55,172 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:55,186 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:56,147 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:24:57,204 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:57,214 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:24:59,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:24:59,274 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:24:59,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:24:59,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:25:00,096 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:00,106 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:00,656 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:01,493 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:02,126 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:02,138 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:05,017 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:05,030 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:06,658 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:07,053 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:07,070 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:07,266 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:09,092 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:09,102 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:11,998 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:12,009 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:12,316 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:12,659 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:14,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:25:14,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:25:14,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:25:15,589 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:25:15,609 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:15,618 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:17,385 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:17,639 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:17,649 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:18,661 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:20,490 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:20,503 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:22,386 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:22,530 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:22,543 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:24,663 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:25,535 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:25,546 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:27,449 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:27,564 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:27,575 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:29,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:25:29,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:25:29,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:25:29,504 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:25:30,578 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:30,596 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:30,665 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:32,506 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:32,623 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:32,637 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:34,664 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:34,679 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:36,667 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:37,423 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:37,435 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:37,553 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:39,455 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:39,466 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:41,497 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:41,508 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:42,594 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:44,235 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:44,246 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:44,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:25:44,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:25:44,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:25:44,669 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:46,266 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:46,275 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:47,649 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:49,129 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:49,139 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:50,671 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:52,649 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:52,720 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:25:52,739 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:52,750 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:55,888 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:55,901 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:56,673 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:25:57,713 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:25:57,919 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:57,931 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:59,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:25:59,273 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:25:59,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:25:59,464 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:25:59,955 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:25:59,965 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:02,674 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:02,763 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:02,867 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:02,885 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:04,923 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:04,933 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:07,806 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:07,825 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:07,833 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:08,677 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:09,850 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:09,864 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:12,905 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:12,973 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:12,996 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:14,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:26:14,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:26:14,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:26:14,679 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:15,022 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:15,035 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:17,952 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:17,999 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:18,014 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:20,052 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:20,062 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:20,681 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:22,923 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:22,935 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:22,953 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:24,955 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:24,969 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:26,683 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:26,988 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:26,999 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:28,006 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:29,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:26:29,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:26:29,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:26:29,422 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:26:30,273 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:26:30,295 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:30,317 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:32,684 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:33,075 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:33,294 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:33,307 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:35,325 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:35,334 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:38,202 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:38,212 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:38,346 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:38,686 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:40,230 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:40,240 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:43,368 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:43,384 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:43,401 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:44,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:26:44,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:26:44,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:26:44,689 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:45,406 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:45,415 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:48,295 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:48,312 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:48,454 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:50,334 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:50,343 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:50,690 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:53,345 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:53,356 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:53,454 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:55,375 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:55,388 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:56,692 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:26:58,495 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:58,506 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:26:58,521 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:26:59,273 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:26:59,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:26:59,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:26:59,440 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:27:00,529 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:00,540 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:02,694 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:03,562 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:03,567 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:03,577 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:05,600 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:05,610 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:08,621 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:08,697 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:09,021 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:27:09,041 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:09,054 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:11,795 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:11,809 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:12,761 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:27:12,762 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:27:12,763 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:27:12,763 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:27:13,698 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:27:13,764 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:13,827 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:13,838 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:14,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:27:14,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:27:14,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:27:14,699 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:16,829 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:16,842 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:18,865 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:18,876 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:19,738 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:20,701 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:20,902 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:20,914 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:23,719 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:23,734 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:24,739 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:25,755 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:25,764 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:26,702 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:28,795 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:28,811 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:29,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:27:29,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:27:29,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:27:29,498 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:27:30,499 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:30,830 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:30,842 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:32,705 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:33,749 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:33,759 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:35,778 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:35,789 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:35,851 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:38,706 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:38,787 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:38,797 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:40,826 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:40,835 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:40,929 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:42,857 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:42,876 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:44,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:27:44,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:27:44,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:27:44,708 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:46,293 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:27:46,328 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:46,338 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:46,407 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:49,038 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:49,062 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:50,710 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:51,106 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:51,119 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:52,027 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:53,952 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:53,971 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:55,992 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:56,006 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:56,711 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:27:57,075 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:27:58,974 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:58,989 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:27:59,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:27:59,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:27:59,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:27:59,479 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:28:01,031 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:01,044 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:02,143 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:02,713 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:04,028 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:04,048 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:06,087 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:06,100 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:07,144 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:08,715 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:09,116 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:09,135 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:11,161 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:11,180 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:12,190 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:14,168 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:14,189 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:14,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:28:14,274 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:28:14,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:28:14,716 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:16,228 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:16,240 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:17,422 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:19,308 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:19,321 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:20,718 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:21,340 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:21,351 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:23,304 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:24,730 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:28:24,760 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:24,772 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:26,721 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:27,667 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:27,700 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:28,343 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:29,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:28:29,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:28:29,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:28:29,439 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:28:29,730 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:29,740 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:32,588 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:32,609 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:32,722 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:33,400 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:34,633 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:34,651 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:37,510 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:37,532 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:38,461 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:38,724 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:39,574 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:39,584 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:42,530 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:42,553 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:43,462 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:44,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:28:44,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:28:44,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:28:44,572 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:44,588 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:44,726 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:47,476 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:47,496 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:48,486 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:49,535 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:49,547 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:50,727 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:52,498 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:52,523 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:53,573 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:54,543 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:54,559 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:56,729 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:28:57,519 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:57,533 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:58,638 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:28:59,274 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:28:59,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:28:59,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:28:59,470 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:28:59,553 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:28:59,565 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:02,731 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:02,976 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:29:02,995 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:03,008 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:03,684 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:05,890 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:05,909 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:07,930 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:07,951 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:08,733 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:09,089 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:10,786 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:10,812 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:12,841 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:12,857 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:14,090 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:14,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:29:14,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:29:14,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:29:15,497 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:15,521 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:17,547 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:17,561 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:19,483 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:20,274 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:20,293 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:22,337 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:22,348 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:22,737 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:24,881 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:25,360 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:25,375 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:26,738 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:27,399 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:27,416 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:29,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:29:29,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:29:29,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:29:29,448 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:29:30,396 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:30,410 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:30,449 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:30,739 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:32,452 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:32,466 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:35,413 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:35,424 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:36,122 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:36,741 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:37,446 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:37,462 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:41,094 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:29:41,116 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:41,127 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:41,376 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:42,743 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:43,999 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:44,012 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:44,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:29:44,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:29:44,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:29:46,032 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:46,046 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:46,420 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:46,503 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:29:46,504 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:29:46,504 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:29:46,505 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:29:46,744 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:29:48,745 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:48,927 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:48,937 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:50,961 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:50,970 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:51,506 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:53,862 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:53,876 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:54,746 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:29:55,898 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:55,907 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:57,482 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:29:58,798 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:58,814 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:29:59,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:29:59,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:29:59,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:29:59,453 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:30:00,748 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:00,836 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:00,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:02,876 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:02,888 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:03,455 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:05,592 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:05,611 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:06,750 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:07,658 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:07,668 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:08,603 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:10,610 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:10,631 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:12,651 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:12,662 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:12,752 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:13,653 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:14,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:30:14,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:30:14,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:30:15,644 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:15,664 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:18,698 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:18,753 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:19,213 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:30:19,234 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:19,243 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:21,269 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:21,279 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:23,757 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:24,076 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:24,090 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:24,755 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:26,110 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:26,120 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:28,758 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:29,222 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:29,235 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:29,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:30:29,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:30:29,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:30:29,403 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:30:30,757 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:31,264 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:31,277 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:34,156 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:34,165 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:34,404 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:36,183 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:36,196 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:36,760 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:39,218 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:39,231 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:39,853 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:41,253 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:41,263 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:42,762 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:44,175 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:44,187 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:44,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:30:44,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:30:44,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:30:45,408 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:46,215 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:46,224 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:48,764 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:49,081 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:49,093 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:50,963 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:51,112 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:51,122 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:53,153 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:53,163 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:54,767 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:30:56,010 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:30:56,431 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:30:56,460 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:56,474 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:59,202 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:59,212 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:30:59,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:30:59,275 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:30:59,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:30:59,439 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:31:00,769 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:01,231 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:01,241 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:01,440 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:04,110 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:04,123 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:06,143 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:06,152 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:06,771 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:07,106 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:09,258 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:09,270 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:11,288 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:11,298 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:12,156 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:12,773 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:14,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:31:14,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:31:14,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:31:14,290 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:14,301 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:16,321 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:16,334 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:17,348 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:18,775 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:19,190 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:19,205 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:21,226 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:21,235 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:22,348 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:24,105 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:24,118 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:24,777 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:26,139 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:26,151 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:27,401 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:28,172 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:28,184 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:29,275 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:31:29,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:31:29,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:31:29,479 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:31:30,779 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:30,924 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:30,935 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:32,480 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:34,823 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:31:34,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:34,859 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:36,782 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:37,490 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:37,868 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:37,878 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:39,899 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:39,912 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:42,542 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:42,779 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:42,784 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:42,789 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:44,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:31:44,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:31:44,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:31:44,811 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:44,824 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:46,846 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:46,862 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:47,586 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:48,786 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:49,678 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:49,690 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:51,710 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:51,724 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:52,586 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:53,752 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:53,763 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:54,788 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:31:56,489 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:56,506 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:57,647 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:31:58,528 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:58,545 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:31:59,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:31:59,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:31:59,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:31:59,529 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:32:00,566 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:00,585 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:00,790 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:02,704 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:03,327 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:03,347 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:05,365 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:05,382 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:06,792 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:07,760 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:08,446 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:08,458 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:11,977 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:32:11,998 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:12,009 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:12,795 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:12,829 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:14,035 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:14,047 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:14,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:32:14,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:32:14,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:32:16,992 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:17,015 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:17,883 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:17,954 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:32:17,956 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:32:17,956 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:32:17,956 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:32:18,797 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:32:18,797 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:19,036 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:19,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:22,264 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:22,276 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:22,957 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:24,299 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:24,311 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:24,799 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:27,212 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:27,224 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:28,947 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:29,243 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:29,252 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:29,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:32:29,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:32:29,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:32:29,486 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:32:30,802 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:32,115 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:32,126 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:34,009 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:34,146 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:34,156 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:36,176 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:36,187 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:36,804 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:38,930 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:38,940 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:39,054 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:40,968 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:40,980 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:43,001 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:43,013 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:44,113 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:44,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:32:44,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:32:44,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:32:44,807 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:45,721 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:45,732 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:49,161 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:49,296 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:32:49,315 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:49,325 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:50,809 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:52,376 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:52,394 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:54,161 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:54,414 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:54,423 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:56,444 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:56,456 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:56,811 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:32:59,201 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:32:59,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:32:59,276 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:32:59,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:32:59,280 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:59,297 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:32:59,400 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:33:01,340 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:01,353 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:02,814 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:03,383 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:03,395 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:04,269 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:06,243 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:06,262 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:08,281 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:08,295 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:08,816 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:09,325 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:11,150 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:11,177 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:13,198 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:13,215 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:14,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:33:14,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:33:14,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:33:14,502 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:14,818 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:15,247 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:15,267 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:18,021 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:18,054 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:19,635 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:20,074 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:20,089 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:20,820 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:23,144 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:23,161 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:24,636 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:26,760 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:33:26,781 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:26,791 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:26,823 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:29,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:33:29,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:33:29,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:33:29,499 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:33:29,938 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:29,955 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:30,500 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:31,975 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:31,995 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:32,825 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:34,038 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:34,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:35,767 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:36,806 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:36,819 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:38,827 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:38,858 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:38,868 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:40,804 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:41,711 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:41,731 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:43,751 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:43,763 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:44,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:33:44,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:33:44,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:33:44,829 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:46,502 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:46,734 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:46,755 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:48,775 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:48,788 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:50,831 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:51,747 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:51,770 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:51,914 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:53,791 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:53,805 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:56,791 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:56,811 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:56,833 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:33:56,975 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:33:58,830 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:58,844 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:33:59,276 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:33:59,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:33:59,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:33:59,475 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:34:01,827 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:01,838 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:02,028 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:02,836 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:05,523 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:34:05,543 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:05,554 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:07,029 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:08,525 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:08,554 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:08,838 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:10,582 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:10,597 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:12,086 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:13,453 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:13,465 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:14,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:34:14,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:34:14,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:34:14,841 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:15,505 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:15,514 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:17,496 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:18,342 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:18,364 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:20,396 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:20,405 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:20,843 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:23,265 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:23,283 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:23,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:25,305 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:25,318 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:26,845 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:28,299 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:28,317 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:28,334 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:29,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:34:29,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:34:29,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:34:29,449 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:34:30,363 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:30,372 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:32,848 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:33,209 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:33,232 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:33,381 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:35,252 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:35,264 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:38,126 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:38,139 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:38,445 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:38,851 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:40,162 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:40,173 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:43,446 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:43,690 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:34:43,709 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:43,719 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:44,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:34:44,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:34:44,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:34:44,853 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:46,477 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:46,498 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:48,521 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:48,537 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:49,428 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:49,618 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:34:49,620 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:34:49,620 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:34:49,621 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:34:49,855 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:34:50,855 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:51,488 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:51,512 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:53,532 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:53,551 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:54,621 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:34:56,518 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:56,541 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:56,858 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:34:58,563 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:58,581 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:34:59,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:34:59,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:34:59,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:34:59,487 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:35:00,489 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:01,436 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:01,463 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:02,860 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:03,495 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:03,521 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:05,667 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:06,427 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:06,450 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:08,488 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:08,499 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:08,862 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:10,708 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:11,442 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:11,455 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:13,473 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:13,489 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:14,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:35:14,277 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:35:14,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:35:14,865 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:16,422 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:16,546 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:16,560 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:18,581 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:18,593 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:20,867 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:21,810 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:22,047 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:35:22,069 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:22,087 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:24,982 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:25,002 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:26,848 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:26,869 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:27,024 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:27,041 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:29,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:35:29,278 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:35:29,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:35:29,411 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:35:29,888 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:29,909 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:31,954 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:31,967 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:32,701 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:32,871 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:34,946 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:34,956 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:36,872 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:36,996 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:37,007 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:38,496 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:38,873 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:40,043 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:40,063 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:40,874 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:42,085 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:42,098 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:43,955 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:44,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:35:44,278 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:35:44,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:35:44,875 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:45,186 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:45,213 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:47,234 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:47,255 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:48,876 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:49,154 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:50,105 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:50,129 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:52,167 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:52,185 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:54,564 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:35:54,878 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:35:55,209 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:55,221 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:57,244 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:57,258 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:35:59,277 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:35:59,278 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:35:59,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:35:59,404 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:35:59,628 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:00,768 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:36:00,787 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:00,804 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:00,880 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:03,610 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:03,622 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:04,629 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:05,664 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:05,673 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:06,882 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:08,495 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:08,515 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:09,701 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:10,547 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:10,558 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:12,883 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:13,506 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:13,528 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:14,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:36:14,278 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:36:14,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:36:15,437 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:15,549 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:15,565 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:18,545 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:18,560 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:18,885 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:20,600 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:20,619 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:20,801 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:23,570 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:23,585 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:24,887 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:25,627 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:25,637 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:25,871 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:28,508 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:28,531 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:29,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:36:29,278 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:36:29,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:36:29,471 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:36:30,552 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:30,571 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:30,888 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:30,920 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:32,605 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:32,616 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:35,443 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:35,453 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:35,921 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:36,890 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:39,051 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:36:39,069 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:39,078 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:40,967 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:42,079 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:42,094 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:42,892 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:44,124 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:44,138 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:44,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:36:44,278 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:36:44,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:36:46,425 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:47,006 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:47,022 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:48,894 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:49,062 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:49,071 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:52,040 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:52,056 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:52,082 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:54,077 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:54,092 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:54,896 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:36:56,948 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:56,967 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:57,133 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:36:58,992 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:59,010 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:36:59,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:36:59,278 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:36:59,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:36:59,425 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:37:00,897 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:01,858 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:01,880 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:02,195 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:03,906 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:03,926 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:06,320 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:37:06,322 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:37:06,322 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:37:06,323 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:37:06,897 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:06,899 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:37:06,899 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:06,918 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:07,323 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:08,899 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:08,941 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:08,958 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:12,016 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:12,029 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:12,900 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:13,299 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:14,049 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:14,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:14,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:37:14,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:37:14,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:37:17,453 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:37:17,476 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:17,487 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:18,347 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:18,902 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:20,435 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:20,454 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:22,497 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:22,510 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:23,348 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:24,904 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:25,392 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:25,413 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:27,455 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:27,465 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:28,399 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:29,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:37:29,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:37:29,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:37:29,451 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:37:30,532 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:30,553 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:30,906 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:32,574 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:32,591 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:33,449 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:35,468 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:35,490 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:37,511 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:37,523 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:38,505 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:38,909 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:40,398 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:40,422 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:42,442 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:42,463 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:43,539 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:44,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:37:44,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:37:44,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:37:44,911 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:45,298 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:45,331 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:47,354 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:47,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:48,589 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:50,410 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:50,421 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:50,913 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:52,441 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:52,453 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:53,589 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:55,895 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:37:55,922 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:55,933 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:56,914 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:37:58,648 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:37:58,841 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:58,861 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:37:59,278 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:37:59,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:37:59,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:37:59,417 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:38:00,886 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:00,906 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:02,916 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:03,683 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:03,872 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:03,894 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:05,916 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:05,935 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:07,959 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:07,974 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:08,732 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:08,918 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:10,850 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:10,872 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:12,904 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:12,915 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:13,797 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:14,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:38:14,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:38:14,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:38:14,920 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:15,902 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:15,926 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:17,954 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:17,978 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:18,852 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:20,922 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:21,054 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:21,074 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:23,125 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:23,138 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:23,853 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:25,972 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:25,993 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:26,924 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:28,011 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:28,021 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:28,905 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:29,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:38:29,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:38:29,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:38:29,418 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:38:31,095 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:31,105 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:32,926 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:33,952 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:34,668 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:38:34,686 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:34,697 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:37,699 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:37,715 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:38,928 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:39,001 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:39,776 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:39,796 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:42,835 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:42,859 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:44,054 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:44,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:38:44,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:38:44,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:38:44,879 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:44,895 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:44,930 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:47,854 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:47,874 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:49,262 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:50,120 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:50,141 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:50,932 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:52,552 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:52,572 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:52,932 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:54,679 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:54,702 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:54,902 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:38:56,933 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:57,653 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:57,675 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:58,934 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:38:59,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:38:59,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:38:59,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:38:59,412 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:38:59,697 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:38:59,710 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:00,370 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:00,934 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:02,599 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:02,619 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:04,652 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:04,663 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:04,936 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:05,832 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:06,936 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:07,650 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:07,664 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:08,937 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:09,685 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:09,695 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:11,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:11,317 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:39:11,319 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:39:11,319 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:39:11,320 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:39:11,938 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:39:12,938 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:14,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:39:14,279 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:39:14,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:39:14,938 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:15,678 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:39:15,704 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:15,717 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:16,431 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:17,740 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:17,752 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:18,941 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:19,776 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:19,788 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:20,942 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:21,486 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:22,533 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:22,545 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:22,942 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:24,564 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:24,577 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:26,944 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:26,947 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:27,655 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:27,687 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:28,944 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:29,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:39:29,280 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:39:29,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:39:29,512 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:39:30,589 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:30,603 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:30,945 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:32,408 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:32,620 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:32,633 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:34,947 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:35,590 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:35,604 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:36,947 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:37,624 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:37,633 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:37,873 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:38,949 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:40,804 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:40,822 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:42,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:42,855 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:42,950 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:43,336 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:44,279 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:39:44,280 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:39:44,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:39:44,951 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:45,898 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:45,907 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:47,928 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:47,942 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:48,790 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:48,953 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:50,873 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:50,885 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:50,953 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:52,954 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:54,256 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:54,518 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:39:54,546 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:54,560 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:56,579 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:56,595 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:56,956 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:58,956 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:39:59,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:39:59,280 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:39:59,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:39:59,417 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:39:59,418 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:39:59,427 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:39:59,437 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:00,957 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:01,456 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:01,470 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:04,713 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:04,731 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:04,958 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:05,133 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:06,750 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:06,760 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:06,959 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:08,960 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:09,881 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:09,892 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:10,581 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:11,916 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:11,929 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:12,961 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:14,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:40:14,280 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:40:14,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:40:14,895 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:14,961 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:14,988 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:16,421 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:17,009 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:17,019 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:18,963 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:19,535 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:40:19,536 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:40:19,537 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:40:19,537 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:40:19,934 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:19,945 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:19,964 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:40:20,965 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:21,538 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:21,969 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:21,979 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:22,965 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:24,383 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:24,399 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:26,966 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:26,967 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:27,279 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:27,292 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:28,968 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:29,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:40:29,280 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:40:29,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:40:29,497 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:40:30,006 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:30,019 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:30,968 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:32,430 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:33,657 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:40:33,680 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:33,691 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:34,970 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:36,268 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:36,281 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:36,970 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:37,897 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:38,971 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:39,214 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:39,224 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:42,182 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:42,200 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:42,972 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:43,357 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:44,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:40:44,281 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:40:44,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:40:44,973 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:45,091 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:45,101 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:47,124 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:47,138 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:48,820 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:48,974 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:50,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:50,069 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:50,975 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:52,086 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:52,097 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:52,976 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:54,120 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:54,130 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:54,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:40:56,966 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:56,977 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:56,978 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:58,978 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:40:58,998 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:59,009 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:40:59,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:40:59,281 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:40:59,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:40:59,424 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:40:59,424 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:00,978 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:02,011 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:02,025 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:04,044 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:04,054 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:04,980 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:05,219 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:06,981 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:06,984 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:07,001 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:08,982 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:09,024 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:09,033 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:10,692 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:12,631 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:41:12,661 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:12,679 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:12,983 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:14,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:41:14,281 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:41:14,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:41:14,984 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:15,629 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:15,640 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:16,460 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:17,661 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:17,672 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:18,985 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:19,699 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:19,712 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:20,986 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:21,625 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:21,742 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:21,753 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:22,986 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:24,584 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:24,596 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:26,621 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:26,636 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:26,987 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:27,089 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:27,839 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:41:27,841 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:41:27,841 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:41:27,842 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:41:27,988 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:41:28,989 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:29,014 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:29,023 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:29,280 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:41:29,281 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:41:29,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:41:29,438 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:41:31,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:31,057 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:31,991 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:32,542 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:34,042 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:34,053 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:35,992 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:36,074 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:36,086 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:37,992 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:38,012 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:38,999 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:39,011 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:39,993 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:41,030 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:41,044 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:43,478 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:43,995 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:44,050 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:44,063 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:44,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:41:44,281 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:41:44,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:41:45,995 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:46,081 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:46,091 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:48,940 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:49,663 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:41:49,685 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:49,705 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:49,996 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:51,996 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:52,475 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:52,486 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:53,997 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:54,404 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:54,506 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:54,518 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:56,552 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:56,565 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:57,998 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:41:59,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:41:59,281 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:41:59,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:41:59,305 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:59,314 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:41:59,463 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:41:59,464 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:41:59,999 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:01,339 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:01,348 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:01,999 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:04,360 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:04,376 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:05,334 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:06,001 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:06,395 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:06,405 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:08,002 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:08,430 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:08,441 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:10,002 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:10,802 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:11,186 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:11,200 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:13,219 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:13,233 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:14,003 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:14,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:42:14,281 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:42:14,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:42:16,005 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:16,254 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:16,264 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:16,449 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:18,283 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:18,295 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:20,006 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:21,312 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:21,326 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:21,734 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:22,006 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:23,348 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:23,363 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:24,007 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:26,934 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:42:26,958 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:26,969 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:27,181 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:28,009 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:29,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:42:29,282 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:42:29,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:42:29,490 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:42:29,776 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:29,797 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:30,010 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:31,816 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:31,825 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:32,010 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:32,647 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:34,799 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:34,811 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:36,012 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:36,132 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:42:36,133 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:42:36,134 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:42:36,134 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:42:36,831 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:36,841 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:37,012 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:42:37,012 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:38,012 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:38,135 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:39,907 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:39,916 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:40,013 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:41,935 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:41,948 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:43,566 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:44,014 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:44,281 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:42:44,282 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:42:44,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:42:44,984 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:44,994 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:46,016 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:47,014 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:47,024 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:49,038 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:50,017 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:50,052 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:50,065 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:52,018 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:52,089 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:52,100 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:54,018 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:54,501 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:42:55,027 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:55,037 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:57,060 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:57,073 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:58,020 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:42:59,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:42:59,282 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:42:59,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:42:59,439 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:42:59,987 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:42:59,996 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:00,021 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:00,440 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:02,021 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:02,022 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:02,030 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:05,422 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:43:05,447 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:05,456 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:06,023 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:06,159 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:08,024 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:08,242 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:08,253 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:10,024 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:10,272 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:10,287 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:11,625 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:13,354 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:13,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:14,026 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:14,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:43:14,282 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:43:14,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:43:15,391 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:15,401 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:16,026 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:17,076 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:18,432 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:18,441 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:20,030 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:20,460 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:20,474 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:22,032 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:22,521 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:23,071 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:23,084 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:24,031 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:25,881 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:25,893 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:27,993 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:28,033 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:28,605 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:28,616 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:29,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:43:29,282 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:43:29,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:43:29,502 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:43:30,034 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:30,640 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:30,651 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:32,034 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:33,452 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:33,547 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:33,557 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:35,577 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:35,589 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:36,035 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:38,036 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:38,637 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:38,654 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:38,920 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:40,037 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:41,649 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:41,669 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:44,038 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:44,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:43:44,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:43:44,283 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:43:44,405 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:44,406 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:43:44,407 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:43:44,407 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:43:44,408 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:43:45,039 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:43:45,039 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:46,039 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:47,144 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:43:47,163 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:47,173 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:49,196 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:49,206 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:49,846 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:50,040 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:52,041 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:52,054 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:52,069 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:54,041 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:54,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:54,378 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:55,312 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:43:56,396 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:56,409 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:58,043 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:43:58,826 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:58,838 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:43:59,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:43:59,282 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:43:59,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:43:59,504 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:44:00,043 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:00,506 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:01,624 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:01,637 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:02,044 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:03,657 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:03,666 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:06,046 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:06,235 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:06,691 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:06,702 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:08,046 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:08,721 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:08,732 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:10,047 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:11,684 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:11,686 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:11,702 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:13,723 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:13,735 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:14,049 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:14,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:44:14,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:44:14,283 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:44:16,049 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:16,672 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:16,684 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:17,138 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:18,710 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:18,732 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:20,050 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:21,757 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:21,767 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:22,051 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:22,603 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:24,051 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:25,424 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:44:25,440 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:25,451 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:27,788 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:28,053 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:28,435 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:28,449 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:29,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:44:29,283 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:44:29,284 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:44:29,490 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:44:30,054 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:30,475 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:30,489 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:33,088 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:33,096 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:33,493 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:35,115 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:35,125 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:37,602 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:37,611 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:38,493 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:39,632 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:39,645 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:42,003 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:42,014 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:43,941 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:44,059 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:44,282 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:44:44,283 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:44:44,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:44:44,871 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:44,882 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:46,060 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:46,901 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:46,916 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:49,359 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:49,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:49,855 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:50,062 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:51,875 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:51,889 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:52,063 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:54,063 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:54,820 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:44:54,931 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:54,945 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:56,965 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:56,975 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:58,064 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:44:59,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:44:59,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:44:59,284 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:44:59,487 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:44:59,861 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:44:59,881 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:00,065 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:00,488 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:03,541 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:45:03,563 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:03,573 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:03,931 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:45:03,932 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:45:03,933 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:45:03,933 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:45:04,067 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:45:04,067 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:05,067 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:05,592 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:05,606 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:05,934 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:06,068 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:08,069 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:08,510 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:08,521 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:10,542 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:10,551 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:11,353 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:12,071 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:13,602 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:13,618 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:14,072 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:14,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:45:14,283 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:45:14,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:45:15,640 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:15,653 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:16,073 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:16,817 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:18,660 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:18,675 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:20,074 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:20,707 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:20,718 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:22,074 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:22,270 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:23,773 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:23,784 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:24,076 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:25,806 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:25,815 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:27,717 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:28,077 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:28,964 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:28,977 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:29,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:45:29,283 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:45:29,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:45:29,481 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:45:30,078 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:30,997 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:31,007 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:33,031 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:33,143 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:34,079 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:34,084 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:36,079 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:36,111 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:36,125 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:38,080 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:38,584 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:39,202 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:39,213 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:42,082 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:42,804 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:45:42,826 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:42,837 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:44,015 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:44,082 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:44,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:45:44,283 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:45:44,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:45:44,861 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:44,872 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:46,083 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:47,598 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:47,607 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:49,458 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:49,631 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:49,644 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:50,084 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:52,085 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:52,847 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:52,860 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:54,085 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:54,878 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:54,892 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:54,899 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:45:57,842 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:57,856 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:58,086 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:45:59,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:45:59,283 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:45:59,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:45:59,428 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:45:59,875 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:45:59,886 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:00,087 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:00,429 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:02,088 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:02,936 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:02,951 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:05,318 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:05,328 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:05,797 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:06,089 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:07,350 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:07,361 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:08,089 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:09,861 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:09,872 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:11,245 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:11,893 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:11,904 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:12,000 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:46:12,001 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:46:12,001 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:46:12,002 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:46:12,090 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:46:12,091 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:13,091 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:14,091 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:14,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:46:14,284 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:46:14,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:46:14,929 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:14,946 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:16,092 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:16,700 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:17,229 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:17,521 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:20,093 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:22,094 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:22,131 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:22,455 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:46:22,478 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:22,495 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:24,095 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:25,613 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:25,631 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:26,095 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:27,593 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:27,651 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:27,661 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:29,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:46:29,284 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:46:29,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:46:29,429 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:46:30,097 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:30,652 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:30,667 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:32,097 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:32,694 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:32,710 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:33,048 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:35,770 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:35,780 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:36,099 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:37,799 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:37,813 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:38,100 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:38,497 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:40,100 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:40,859 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:40,954 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:42,974 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:42,984 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:43,940 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:44,102 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:44,283 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:46:44,393 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:46:44,394 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:46:45,931 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:45,943 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:46,102 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:47,965 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:47,978 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:48,103 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:49,386 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:50,025 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:50,048 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:52,071 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:52,086 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:52,104 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:54,105 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:54,113 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:54,129 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:54,833 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:46:56,106 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:46:56,157 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:56,168 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:46:59,284 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:46:59,284 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:46:59,285 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:46:59,472 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:47:00,107 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:00,473 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:01,088 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:47:01,115 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:01,131 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:02,108 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:04,025 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:04,187 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:05,756 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:06,109 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:06,208 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:06,225 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:08,110 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:08,676 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:08,693 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:10,110 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:11,197 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:11,487 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:11,503 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:13,527 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:13,538 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:14,112 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:14,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:47:14,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:47:14,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:47:15,832 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:15,843 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:16,113 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:16,637 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:18,034 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:18,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:18,113 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:20,066 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:20,082 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:20,135 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:47:20,136 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:47:20,136 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:47:20,137 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:47:21,114 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:47:22,114 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:22,138 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:22,599 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:22,609 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:24,114 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:25,488 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:25,502 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:26,115 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:27,524 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:27,534 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:27,570 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:29,284 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:47:29,285 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:47:29,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:47:29,451 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:47:30,116 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:30,464 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:30,478 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:32,117 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:32,500 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:32,510 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:33,027 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:34,118 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:35,560 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:35,576 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:38,119 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:38,487 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:39,358 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:47:39,380 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:39,392 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:40,119 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:42,318 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:42,331 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:43,934 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:44,121 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:44,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:47:44,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:47:44,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:47:44,352 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:44,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:46,122 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:47,283 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:47,295 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:48,122 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:49,315 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:49,329 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:49,379 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:52,124 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:52,265 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:52,282 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:54,125 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:54,305 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:54,317 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:54,827 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:47:56,126 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:47:56,699 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:56,709 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:58,731 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:58,741 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:47:59,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:47:59,285 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:47:59,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:47:59,451 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:48:00,128 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:00,452 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:01,699 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:01,712 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:02,128 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:03,731 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:03,742 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:04,129 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:05,743 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:06,756 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:06,767 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:08,130 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:08,787 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:08,799 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:10,131 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:11,204 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:11,702 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:11,715 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:13,734 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:13,748 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:14,132 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:14,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:48:14,285 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:48:14,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:48:16,133 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:16,652 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:17,299 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:48:17,321 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:17,336 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:18,133 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:20,191 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:20,203 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:22,110 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:22,136 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:22,221 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:22,236 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:24,136 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:24,261 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:24,277 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:26,138 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:27,110 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:27,122 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:27,573 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:28,328 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:48:28,330 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:48:28,330 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:48:28,331 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:48:29,139 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:48:29,147 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:29,157 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:29,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:48:29,285 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:48:29,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:48:29,494 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:48:30,139 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:32,141 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:32,175 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:32,186 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:33,035 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:34,142 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:34,205 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:34,216 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:37,262 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:37,271 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:38,143 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:38,502 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:39,292 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:39,308 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:40,144 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:42,419 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:42,430 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:43,962 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:44,145 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:44,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:48:44,285 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:48:44,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:48:44,450 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:44,468 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:46,146 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:47,396 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:47,408 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:48,147 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:49,425 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:49,427 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:49,437 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:52,149 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:52,478 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:52,490 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:54,149 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:54,875 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:48:56,057 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:48:56,076 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:56,092 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:56,150 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:48:58,125 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:58,135 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:48:59,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:48:59,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:48:59,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:48:59,430 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:49:00,151 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:00,431 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:00,957 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:00,967 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:02,152 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:02,991 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:03,005 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:04,153 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:05,793 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:05,923 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:05,935 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:07,955 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:07,974 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:08,154 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:10,155 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:10,874 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:10,885 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:11,252 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:12,907 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:12,918 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:14,156 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:14,285 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:49:14,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:49:14,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:49:15,921 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:15,932 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:16,158 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:16,712 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:17,952 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:17,962 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:18,158 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:21,019 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:21,030 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:22,159 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:22,172 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:23,052 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:23,063 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:24,160 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:25,965 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:25,979 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:26,160 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:27,626 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:27,999 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:28,008 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:29,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:49:29,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:49:29,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:49:29,408 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:49:30,162 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:31,009 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:31,019 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:32,162 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:33,069 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:34,163 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:34,672 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:49:34,690 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:34,700 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:36,554 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:49:36,555 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:49:36,555 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:49:36,556 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:49:37,164 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:49:37,702 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:37,713 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:38,164 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:38,557 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:39,731 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:39,746 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:40,165 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:42,802 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:42,813 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:43,977 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:44,166 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:44,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:49:44,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:49:44,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:49:44,834 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:44,846 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:46,167 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:47,753 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:47,765 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:48,167 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:49,437 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:49,785 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:49,795 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:51,817 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:51,830 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:52,169 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:54,169 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:54,766 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:54,779 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:54,878 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:49:56,170 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:49:56,799 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:56,813 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:58,836 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:49:59,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:49:59,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:49:59,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:49:59,503 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:49:59,980 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:00,505 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:01,171 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:02,297 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:02,307 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:03,172 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:04,325 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:04,336 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:05,172 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:05,778 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:06,810 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:06,821 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:08,847 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:08,857 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:09,174 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:11,175 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:11,220 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:13,176 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:14,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:50:14,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:50:14,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:50:14,450 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:50:14,472 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:14,482 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:16,666 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:17,177 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:17,553 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:17,564 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:19,178 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:19,944 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:19,957 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:21,978 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:21,991 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:22,131 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:23,179 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:25,003 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:25,013 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:25,180 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:27,034 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:27,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:27,181 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:27,593 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:29,286 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:50:29,286 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:50:29,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:50:29,506 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:50:30,185 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:30,195 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:31,182 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:32,215 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:32,227 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:33,054 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:33,183 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:34,248 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:35,184 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:35,297 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:37,319 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:37,329 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:38,505 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:39,185 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:39,353 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:40,290 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:41,185 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:42,310 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:42,321 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:43,186 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:43,935 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:44,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:50:44,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:50:44,288 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:50:44,344 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:44,353 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:44,688 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:50:44,690 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:50:44,690 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:50:44,690 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:50:45,186 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:50:46,187 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:47,187 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:47,484 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:47,497 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:49,187 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:49,392 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:49,559 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:49,570 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:53,189 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:54,839 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:50:55,190 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:56,257 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:50:56,277 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:56,290 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:57,190 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:50:58,318 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:58,329 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:50:59,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:50:59,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:50:59,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:50:59,446 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:51:00,447 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:01,191 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:01,272 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:01,283 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:03,192 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:03,303 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:03,316 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:05,193 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:05,741 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:06,317 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:06,335 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:08,353 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:08,363 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:09,194 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:11,186 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:11,195 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:11,421 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:11,432 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:13,195 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:13,454 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:13,468 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:14,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:51:14,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:51:14,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:51:16,385 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:16,396 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:16,621 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:17,197 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:18,427 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:18,440 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:19,198 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:21,198 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:21,544 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:21,554 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:22,085 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:23,573 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:23,585 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:25,200 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:26,564 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:26,578 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:27,201 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:27,532 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:28,599 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:28,609 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:29,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:51:29,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:51:29,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:51:29,459 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:51:31,202 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:31,394 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:31,406 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:32,974 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:33,202 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:34,770 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:51:34,794 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:34,806 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:35,203 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:37,300 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:37,312 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:38,434 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:39,204 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:39,794 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:39,806 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:41,206 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:42,747 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:42,761 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:43,206 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:43,894 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:44,287 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:51:44,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:51:44,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:51:44,781 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:44,793 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:47,207 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:47,939 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:47,949 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:49,208 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:49,334 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:49,970 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:49,985 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:51,209 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:52,823 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:51:52,824 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:51:52,825 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:51:52,825 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:51:52,900 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:52,909 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:53,209 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:51:54,209 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:54,826 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:51:54,930 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:54,944 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:55,210 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:57,210 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:51:57,581 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:57,594 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:51:59,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:51:59,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:51:59,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:51:59,438 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:52:00,034 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:00,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:00,438 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:01,211 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:02,319 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:02,331 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:03,212 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:04,659 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:04,677 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:05,213 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:05,713 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:07,469 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:07,481 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:09,214 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:09,504 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:09,518 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:11,169 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:11,214 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:13,196 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:52:13,215 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:13,223 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:13,234 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:14,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:52:14,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:52:14,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:52:16,119 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:16,129 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:16,642 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:17,216 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:18,155 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:18,165 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:19,217 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:21,187 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:21,202 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:21,217 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:22,111 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:23,223 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:23,233 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:25,218 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:26,279 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:26,290 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:27,219 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:27,578 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:28,752 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:28,763 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:29,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:52:29,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:52:29,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:52:29,489 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:52:30,782 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:30,799 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:31,220 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:33,021 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:33,198 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:33,208 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:33,220 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:35,221 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:35,228 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:35,238 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:38,297 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:38,310 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:38,482 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:39,222 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:40,759 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:40,776 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:41,222 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:42,797 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:42,807 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:43,223 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:43,951 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:44,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:52:44,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:52:44,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:52:45,251 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:45,266 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:47,224 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:48,103 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:48,119 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:49,224 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:49,417 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:51,225 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:51,816 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:52:51,845 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:51,854 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:54,883 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:52:54,994 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:55,007 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:55,226 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:57,026 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:57,037 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:52:57,227 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:52:59,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:52:59,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:52:59,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:52:59,463 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:53:00,073 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:00,088 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:00,464 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:01,085 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:53:01,086 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:53:01,086 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:53:01,087 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:53:01,228 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:53:01,229 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:02,111 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:02,125 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:02,229 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:03,229 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:05,183 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:05,198 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:05,229 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:05,782 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:07,219 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:07,230 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:09,231 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:10,277 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:10,296 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:11,227 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:11,231 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:12,316 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:12,326 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:13,231 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:14,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:53:14,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:53:14,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:53:14,350 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:14,364 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:16,388 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:16,410 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:16,686 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:17,232 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:18,429 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:18,439 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:19,233 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:21,233 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:21,544 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:21,554 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:22,150 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:23,576 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:23,591 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:25,234 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:25,615 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:25,628 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:27,235 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:27,603 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:29,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:53:29,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:53:29,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:53:29,520 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:53:31,107 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:53:31,138 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:31,157 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:31,236 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:33,069 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:33,236 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:33,996 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:34,007 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:35,237 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:36,031 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:36,043 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:38,519 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:39,139 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:39,151 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:39,238 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:41,172 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:41,182 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:41,239 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:43,214 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:43,227 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:43,239 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:43,984 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:44,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:53:44,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:53:44,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:53:45,986 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:45,999 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:47,241 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:48,020 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:48,029 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:49,241 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:49,442 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:51,172 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:51,188 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:51,242 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:53,209 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:53,219 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:54,904 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:53:55,242 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:56,391 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:56,407 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:57,243 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:53:58,431 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:58,441 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:53:59,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:53:59,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:53:59,289 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:53:59,453 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:54:00,454 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:01,244 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:01,456 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:01,470 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:03,245 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:03,491 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:03,503 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:05,245 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:05,836 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:06,658 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:06,668 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:09,246 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:09,325 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:54:09,326 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:54:09,327 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:54:09,327 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:54:10,247 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:54:10,247 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:10,424 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:54:10,442 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:10,453 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:11,247 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:11,328 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:13,248 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:13,513 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:13,526 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:14,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:54:14,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:54:14,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:54:15,546 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:15,555 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:16,763 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:17,248 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:18,604 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:18,621 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:19,249 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:20,640 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:20,653 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:21,250 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:22,230 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:22,676 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:22,686 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:25,251 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:25,551 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:25,563 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:27,251 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:27,583 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:27,597 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:27,686 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:29,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:54:29,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:54:29,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:54:29,481 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:54:29,620 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:29,632 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:31,252 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:32,542 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:32,552 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:33,140 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:33,252 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:34,572 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:34,581 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:35,253 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:37,728 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:37,747 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:38,615 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:39,255 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:39,765 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:39,778 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:41,255 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:42,826 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:42,838 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:43,256 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:44,080 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:44,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:54:44,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:54:44,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:54:44,854 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:44,865 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:47,257 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:54:48,094 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:54:48,113 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:48,122 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:49,548 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:50,590 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:50,603 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:52,622 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:52,631 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:54,548 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:54:55,296 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:55,307 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:57,325 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:57,335 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:54:59,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:54:59,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:54:59,291 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:54:59,489 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:55:00,399 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:00,410 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:00,490 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:01,260 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:02,430 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:02,442 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:03,260 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:05,261 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:05,505 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:05,518 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:06,344 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:07,538 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:07,550 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:09,262 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:09,572 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:09,586 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:11,263 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:11,787 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:12,437 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:12,447 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:14,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:55:14,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:55:14,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:55:14,465 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:14,479 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:15,264 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:16,513 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:16,524 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:17,224 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:17,264 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:19,265 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:19,418 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:19,434 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:21,455 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:21,467 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:22,678 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:23,266 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:25,038 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:55:25,058 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:25,067 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:25,266 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:27,267 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:27,998 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:28,011 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:28,103 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:28,848 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:55:28,849 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:55:28,849 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:55:28,850 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:55:29,267 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:55:29,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:55:29,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:55:29,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:55:29,464 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:55:30,033 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:30,043 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:30,267 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:31,268 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:32,069 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:32,085 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:33,268 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:33,544 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:34,943 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:34,953 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:35,269 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:36,976 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:36,987 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:38,996 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:39,008 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:39,023 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:39,269 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:41,270 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:41,891 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:41,902 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:43,922 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:43,933 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:44,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:55:44,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:55:44,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:55:44,487 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:45,271 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:45,956 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:45,969 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:47,271 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:48,820 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:48,830 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:49,272 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:49,891 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:50,850 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:50,863 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:52,888 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:52,909 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:53,273 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:55,274 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:55,330 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:55:55,734 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:55,744 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:57,274 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:55:57,768 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:57,784 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:55:59,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:55:59,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:55:59,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:55:59,476 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:56:00,477 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:01,275 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:01,538 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:56:01,559 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:01,570 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:03,276 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:04,507 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:04,519 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:05,277 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:06,250 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:06,797 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:06,809 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:08,830 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:08,840 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:09,278 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:10,859 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:10,873 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:11,279 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:11,692 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:13,280 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:13,819 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:13,831 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:14,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:56:14,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:56:14,290 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:56:15,852 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:15,866 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:17,137 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:17,281 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:17,892 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:17,904 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:19,281 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:20,795 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:20,808 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:22,578 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:22,829 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:22,839 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:23,282 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:25,283 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:25,940 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:25,951 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:27,283 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:27,972 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:27,985 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:28,039 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:29,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:56:29,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:56:29,291 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:56:29,488 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:56:30,981 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:30,994 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:31,285 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:33,013 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:33,022 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:33,285 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:33,462 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:35,065 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:35,079 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:35,286 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:36,929 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:56:36,930 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:56:36,930 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:56:36,931 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:56:37,286 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:56:38,287 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:38,722 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:56:38,749 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:38,761 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:38,932 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:39,287 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:41,287 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:41,727 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:41,737 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:43,288 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:43,758 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:43,771 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:44,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:56:44,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:56:44,291 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:56:44,424 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:46,888 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:46,900 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:47,289 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:48,919 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:48,933 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:49,290 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:49,805 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:50,953 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:50,967 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:53,290 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:53,851 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:53,862 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:55,264 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:56:55,291 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:55,881 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:55,891 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:57,291 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:56:58,916 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:58,927 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:56:59,289 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:56:59,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:56:59,291 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:56:59,499 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:57:00,500 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:00,947 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:00,960 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:01,293 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:02,982 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:02,992 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:03,294 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:05,294 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:05,864 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:05,877 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:06,174 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:07,897 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:07,906 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:09,296 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:09,929 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:09,940 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:11,296 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:11,639 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:12,771 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:12,780 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:13,297 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:14,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:57:14,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:57:14,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:57:16,460 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:57:16,482 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:16,496 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:17,108 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:17,298 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:18,515 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:18,525 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:19,299 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:21,390 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:21,402 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:22,559 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:23,300 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:23,739 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:23,973 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:25,301 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:25,992 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:26,004 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:27,301 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:28,025 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:28,033 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:28,049 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:29,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:57:29,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:57:29,291 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:57:29,519 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:57:30,900 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:30,910 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:31,302 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:32,929 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:32,940 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:33,303 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:33,482 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:35,304 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:36,016 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:36,027 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:38,045 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:38,061 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:38,926 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:39,305 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:41,181 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:41,197 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:41,306 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:43,224 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:43,240 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:43,306 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:44,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:57:44,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:57:44,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:57:44,481 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:45,140 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:57:45,141 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:57:45,142 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:57:45,142 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:57:45,307 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:57:46,277 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:46,297 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:46,307 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:47,308 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:48,320 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:48,331 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:49,308 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:49,830 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:50,352 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:50,368 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:51,309 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:54,249 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:57:54,274 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:54,286 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:55,288 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:57:55,310 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:57,311 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:57:57,655 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:57,666 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:59,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:57:59,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:57:59,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:57:59,455 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:57:59,688 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:57:59,706 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:00,457 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:01,312 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:01,726 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:01,736 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:03,313 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:04,724 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:04,735 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:05,313 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:06,216 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:06,757 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:06,768 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:08,789 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:08,800 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:09,314 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:10,822 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:10,843 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:11,315 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:11,684 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:13,316 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:13,991 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:14,007 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:14,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:58:14,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:58:14,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:58:16,029 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:16,040 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:17,142 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:17,317 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:18,059 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:18,074 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:19,317 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:20,991 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:21,001 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:21,318 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:22,604 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:23,025 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:23,040 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:25,319 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:26,350 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:26,367 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:27,320 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:28,060 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:28,388 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:28,405 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:29,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:58:29,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:58:29,291 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:58:29,524 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:58:31,321 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:32,199 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:58:32,225 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:32,236 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:33,322 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:33,520 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:34,260 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:34,274 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:35,322 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:37,243 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:37,263 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:38,988 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:39,304 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:39,324 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:39,333 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:41,325 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:42,239 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:42,251 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:43,326 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:44,273 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:44,287 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:44,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:58:44,292 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:58:44,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:58:44,439 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:46,307 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:46,326 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:47,327 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:49,232 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:49,243 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:49,328 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:49,933 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:51,267 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:51,279 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:51,328 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:53,298 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:53,310 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:53,422 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 13:58:53,423 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 13:58:53,424 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 13:58:53,424 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 13:58:54,329 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 13:58:54,395 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:55,329 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:55,425 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:58:56,147 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:56,333 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:57,330 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:58,352 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:58,363 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:58:59,290 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:58:59,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:58:59,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:58:59,331 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:58:59,426 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:59:00,427 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:01,264 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:01,275 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:03,296 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:03,307 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:03,333 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:05,331 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:05,333 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:05,348 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:06,321 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:07,334 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:09,010 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:59:09,037 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:09,060 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:11,335 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:11,797 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:12,176 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:12,187 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:13,336 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:14,206 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:14,218 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:14,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:59:14,292 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:59:14,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:59:16,239 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:16,249 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:17,250 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:17,338 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:19,182 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:19,192 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:19,338 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:21,215 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:21,225 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:21,339 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:22,715 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:24,252 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:24,262 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:25,340 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:26,280 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:26,291 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:28,175 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:28,341 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:29,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:59:29,292 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:59:29,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:59:29,306 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:29,317 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:29,509 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:59:30,341 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:31,337 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:31,348 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:33,385 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:33,401 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:33,640 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:34,343 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:36,156 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:36,169 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:36,344 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:38,189 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:38,198 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:38,344 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:39,097 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:41,479 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:41,489 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:42,346 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:43,509 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:43,522 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:44,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:59:44,293 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:59:44,293 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:59:44,294 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:44,346 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:47,124 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 13:59:47,151 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:47,163 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:48,348 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:49,922 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:49,932 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:50,021 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:50,349 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:51,953 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:51,964 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:52,350 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:54,877 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:54,887 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:55,480 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 13:59:56,351 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:56,905 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:56,917 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:58,351 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 13:59:59,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 13:59:59,292 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 13:59:59,292 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 13:59:59,497 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 13:59:59,814 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 13:59:59,824 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:00,352 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 14:00:00,500 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 14:00:01,696 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 14:00:01,746 DEBUG SenderThread:1390067 [sender.py:send():382] send: history +2024-03-27 14:00:01,746 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: summary_record +2024-03-27 14:00:01,746 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 14:00:01,843 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:01,857 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:02,353 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 14:00:02,353 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 14:00:04,353 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 14:00:04,593 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:04,603 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:05,747 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 14:00:06,621 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:06,632 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:09,234 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:09,243 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:10,748 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 14:00:11,264 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:11,274 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:13,833 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:13,843 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:14,291 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 14:00:14,292 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 14:00:14,293 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: stop_status +2024-03-27 14:00:15,862 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:15,872 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:16,356 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 14:00:16,441 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 14:00:18,389 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:18,398 ERROR gpu :1390067 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 14:00:19,694 DEBUG SenderThread:1390067 [sender.py:send():382] send: exit +2024-03-27 14:00:19,694 INFO SenderThread:1390067 [sender.py:send_exit():589] handling exit code: 1 +2024-03-27 14:00:19,694 INFO SenderThread:1390067 [sender.py:send_exit():591] handling runtime: 3710 +2024-03-27 14:00:19,695 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 14:00:19,696 INFO SenderThread:1390067 [sender.py:send_exit():597] send defer +2024-03-27 14:00:19,696 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:19,696 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-27 14:00:19,696 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:19,696 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-27 14:00:19,696 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 1 +2024-03-27 14:00:19,696 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:19,697 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-27 14:00:19,697 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:19,697 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-27 14:00:19,697 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 2 +2024-03-27 14:00:19,697 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:19,697 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-27 14:00:19,697 INFO HandlerThread:1390067 [system_monitor.py:finish():203] Stopping system monitor +2024-03-27 14:00:19,697 DEBUG SystemMonitor:1390067 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-27 14:00:19,698 INFO HandlerThread:1390067 [interfaces.py:finish():202] Joined cpu monitor +2024-03-27 14:00:19,698 DEBUG SystemMonitor:1390067 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-27 14:00:19,698 INFO HandlerThread:1390067 [interfaces.py:finish():202] Joined disk monitor +2024-03-27 14:00:20,358 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 14:00:20,415 ERROR gpu :1390067 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 14:00:20,415 INFO HandlerThread:1390067 [interfaces.py:finish():202] Joined gpu monitor +2024-03-27 14:00:20,415 INFO HandlerThread:1390067 [interfaces.py:finish():202] Joined memory monitor +2024-03-27 14:00:20,415 INFO HandlerThread:1390067 [interfaces.py:finish():202] Joined network monitor +2024-03-27 14:00:20,416 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:20,416 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-27 14:00:20,416 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 3 +2024-03-27 14:00:20,416 DEBUG SenderThread:1390067 [sender.py:send():382] send: stats +2024-03-27 14:00:20,416 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:20,417 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-27 14:00:20,417 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:20,417 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-27 14:00:20,417 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 4 +2024-03-27 14:00:20,417 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:20,417 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-27 14:00:20,417 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:20,417 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-27 14:00:20,417 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 5 +2024-03-27 14:00:20,417 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:20,418 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-27 14:00:20,418 DEBUG SenderThread:1390067 [sender.py:send():382] send: summary +2024-03-27 14:00:20,418 INFO SenderThread:1390067 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 14:00:20,418 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:20,418 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-27 14:00:20,418 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 6 +2024-03-27 14:00:20,418 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:20,418 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-27 14:00:20,419 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:20,419 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-27 14:00:20,419 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 7 +2024-03-27 14:00:20,419 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 14:00:20,419 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:20,419 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-27 14:00:20,419 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:20,419 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-27 14:00:20,694 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-27 14:00:21,358 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 14:00:22,030 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 8 +2024-03-27 14:00:22,030 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: poll_exit +2024-03-27 14:00:22,030 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:22,030 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-27 14:00:22,030 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:22,030 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-27 14:00:22,031 INFO SenderThread:1390067 [job_builder.py:build():296] Attempting to build job artifact +2024-03-27 14:00:22,032 INFO SenderThread:1390067 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-27 14:00:22,053 INFO SenderThread:1390067 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-27 14:00:22,070 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 9 +2024-03-27 14:00:22,071 DEBUG SenderThread:1390067 [sender.py:send():382] send: artifact +2024-03-27 14:00:22,072 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:22,073 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-27 14:00:22,358 INFO Thread-13 :1390067 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 14:00:22,695 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-27 14:00:22,901 INFO wandb-upload_0:1390067 [upload_job.py:push():86] Skipped uploading /home/sanchit/.local/share/wandb/artifacts/staging/tmpcau1hp24 +2024-03-27 14:00:23,173 INFO wandb-upload_1:1390067 [upload_job.py:push():89] Uploaded file /home/sanchit/.local/share/wandb/artifacts/staging/tmpzy9fd12i +2024-03-27 14:00:24,213 INFO SenderThread:1390067 [sender.py:send_artifact():1494] sent artifact job-https___huggingface.co_sanchit-gandhi_distil-large-v3-hi-ft_run_speech_recognition_seq2seq.py - {'id': 'QXJ0aWZhY3Q6NzcyOTA3NzYz', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE1MzEyNTQyNg==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzcyODMxMDMw', 'versionIndex': 0}}} +2024-03-27 14:00:24,213 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:24,213 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-27 14:00:24,213 INFO SenderThread:1390067 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-27 14:00:24,359 INFO SenderThread:1390067 [dir_watcher.py:finish():388] scan: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files +2024-03-27 14:00:24,360 INFO SenderThread:1390067 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/requirements.txt requirements.txt +2024-03-27 14:00:24,360 INFO SenderThread:1390067 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-metadata.json wandb-metadata.json +2024-03-27 14:00:24,360 INFO SenderThread:1390067 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json wandb-summary.json +2024-03-27 14:00:24,360 INFO SenderThread:1390067 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log output.log +2024-03-27 14:00:24,360 INFO SenderThread:1390067 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/config.yaml config.yaml +2024-03-27 14:00:24,360 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 10 +2024-03-27 14:00:24,363 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: poll_exit +2024-03-27 14:00:24,364 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:24,368 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-27 14:00:24,372 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:24,373 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-27 14:00:24,373 INFO SenderThread:1390067 [file_pusher.py:finish():175] shutting down file pusher +2024-03-27 14:00:24,696 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-27 14:00:24,697 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: poll_exit +2024-03-27 14:00:24,715 INFO wandb-upload_0:1390067 [upload_job.py:push():131] Uploaded file /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/requirements.txt +2024-03-27 14:00:24,737 INFO wandb-upload_1:1390067 [upload_job.py:push():131] Uploaded file /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/wandb-summary.json +2024-03-27 14:00:24,764 INFO wandb-upload_2:1390067 [upload_job.py:push():131] Uploaded file /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/output.log +2024-03-27 14:00:24,829 INFO wandb-upload_3:1390067 [upload_job.py:push():131] Uploaded file /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/files/config.yaml +2024-03-27 14:00:25,029 INFO Thread-12 :1390067 [sender.py:transition_state():617] send defer: 11 +2024-03-27 14:00:25,030 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:25,030 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-27 14:00:25,030 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:25,030 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-27 14:00:25,030 INFO SenderThread:1390067 [file_pusher.py:join():181] waiting for file pusher +2024-03-27 14:00:25,030 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 12 +2024-03-27 14:00:25,030 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:25,031 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-27 14:00:25,031 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:25,031 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-27 14:00:25,031 INFO SenderThread:1390067 [file_stream.py:finish():595] file stream finish called +2024-03-27 14:00:25,417 INFO SenderThread:1390067 [file_stream.py:finish():599] file stream finish is done +2024-03-27 14:00:25,417 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 13 +2024-03-27 14:00:25,418 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:25,418 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-27 14:00:25,418 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:25,418 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-27 14:00:25,418 INFO SenderThread:1390067 [sender.py:transition_state():617] send defer: 14 +2024-03-27 14:00:25,418 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: defer +2024-03-27 14:00:25,418 INFO HandlerThread:1390067 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-27 14:00:25,418 DEBUG SenderThread:1390067 [sender.py:send():382] send: final +2024-03-27 14:00:25,419 DEBUG SenderThread:1390067 [sender.py:send():382] send: footer +2024-03-27 14:00:25,419 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: defer +2024-03-27 14:00:25,419 INFO SenderThread:1390067 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-27 14:00:25,419 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-27 14:00:25,420 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: poll_exit +2024-03-27 14:00:25,420 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-27 14:00:25,420 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: poll_exit +2024-03-27 14:00:25,420 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: server_info +2024-03-27 14:00:25,421 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: get_summary +2024-03-27 14:00:25,421 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: server_info +2024-03-27 14:00:25,422 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-27 14:00:25,424 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 14:00:25,424 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: job_info +2024-03-27 14:00:25,554 DEBUG SenderThread:1390067 [sender.py:send_request():409] send_request: job_info +2024-03-27 14:00:25,554 INFO MainThread:1390067 [wandb_run.py:_footer_history_summary_info():3857] rendering history +2024-03-27 14:00:25,555 INFO MainThread:1390067 [wandb_run.py:_footer_history_summary_info():3889] rendering summary +2024-03-27 14:00:25,555 INFO MainThread:1390067 [wandb_run.py:_footer_sync_info():3816] logging synced files +2024-03-27 14:00:25,556 DEBUG HandlerThread:1390067 [handler.py:handle_request():146] handle_request: shutdown +2024-03-27 14:00:25,556 INFO HandlerThread:1390067 [handler.py:finish():869] shutting down handler +2024-03-27 14:00:26,424 INFO WriterThread:1390067 [datastore.py:close():296] close: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_125828-wlmqyk6v/run-wlmqyk6v.wandb +2024-03-27 14:00:26,554 INFO SenderThread:1390067 [sender.py:finish():1572] shutting down sender +2024-03-27 14:00:26,554 INFO SenderThread:1390067 [file_pusher.py:finish():175] shutting down file pusher +2024-03-27 14:00:26,554 INFO SenderThread:1390067 [file_pusher.py:join():181] waiting for file pusher