diff --git "a/wandb/run-20240327_190418-lwtbcr8s/logs/debug-internal.log" "b/wandb/run-20240327_190418-lwtbcr8s/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240327_190418-lwtbcr8s/logs/debug-internal.log" @@ -0,0 +1,9954 @@ +2024-03-27 19:04:18,566 INFO StreamThr :1893870 [internal.py:wandb_internal():86] W&B internal server running at pid: 1893870, started at: 2024-03-27 19:04:18.565290 +2024-03-27 19:04:18,567 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status +2024-03-27 19:04:18,570 INFO WriterThread:1893870 [datastore.py:open_for_write():87] open: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/run-lwtbcr8s.wandb +2024-03-27 19:04:18,570 DEBUG SenderThread:1893870 [sender.py:send():382] send: header +2024-03-27 19:04:18,640 DEBUG SenderThread:1893870 [sender.py:send():382] send: run +2024-03-27 19:04:18,989 INFO SenderThread:1893870 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files +2024-03-27 19:04:18,989 INFO SenderThread:1893870 [sender.py:_start_run_threads():1136] run started: lwtbcr8s with start time 1711562658.56459 +2024-03-27 19:04:18,996 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: check_version +2024-03-27 19:04:18,996 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: check_version +2024-03-27 19:04:19,037 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: run_start +2024-03-27 19:04:19,050 DEBUG HandlerThread:1893870 [system_info.py:__init__():27] System info init +2024-03-27 19:04:19,050 DEBUG HandlerThread:1893870 [system_info.py:__init__():42] System info init done +2024-03-27 19:04:19,050 INFO HandlerThread:1893870 [system_monitor.py:start():194] Starting system monitor +2024-03-27 19:04:19,050 INFO SystemMonitor:1893870 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-27 19:04:19,051 INFO HandlerThread:1893870 [system_monitor.py:probe():214] Collecting system info +2024-03-27 19:04:19,051 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started cpu monitoring +2024-03-27 19:04:19,052 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started disk monitoring +2024-03-27 19:04:19,053 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started gpu monitoring +2024-03-27 19:04:19,054 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started memory monitoring +2024-03-27 19:04:19,056 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started network monitoring +2024-03-27 19:04:19,090 DEBUG HandlerThread:1893870 [system_info.py:probe():151] Probing system +2024-03-27 19:04:19,096 DEBUG HandlerThread:1893870 [system_info.py:_probe_git():136] Probing git +2024-03-27 19:04:19,107 DEBUG HandlerThread:1893870 [system_info.py:_probe_git():144] Probing git done +2024-03-27 19:04:19,107 DEBUG HandlerThread:1893870 [system_info.py:probe():199] Probing system done +2024-03-27 19:04:19,108 DEBUG HandlerThread:1893870 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-03-27T18:04:19.090782', 'startedAt': '2024-03-27T18:04:18.560848', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=distil-whisper/distil-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_1', '--dataset_config_name=hi', '--language=hindi', '--train_split_name=train+validation', '--eval_split_name=test', '--max_steps=5000', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=32', '--logging_steps=25', '--learning_rate=1e-4', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--save_total_limit=1', '--generation_max_length=225', '--preprocessing_num_workers=1', '--dataloader_num_workers=4', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--gradient_checkpointing', '--fp16', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-large-v3-hi-ft', 'commit': '40c686df113c0e98e7363c1bd523f58d11848fc0'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/distil-large-v3-hi-ft', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2290.4996484375006, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2186.729, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.542, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.045, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.339, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.496, 'min': 1500.0, 'max': 2250.0}, {'current': 1742.02, 'min': 1500.0, 'max': 2250.0}, {'current': 1692.095, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.911, 'min': 1500.0, 'max': 2250.0}, {'current': 1606.212, 'min': 1500.0, 'max': 2250.0}, {'current': 1606.808, 'min': 1500.0, 'max': 2250.0}, {'current': 1606.448, 'min': 1500.0, 'max': 2250.0}, {'current': 2094.446, 'min': 1500.0, 'max': 2250.0}, {'current': 1734.944, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.215, 'min': 1500.0, 'max': 2250.0}, {'current': 2489.415, 'min': 1500.0, 'max': 2250.0}, {'current': 1737.217, 'min': 1500.0, 'max': 2250.0}, {'current': 1695.232, 'min': 1500.0, 'max': 2250.0}, {'current': 2416.542, 'min': 1500.0, 'max': 2250.0}, {'current': 1737.614, 'min': 1500.0, 'max': 2250.0}, {'current': 1599.452, 'min': 1500.0, 'max': 2250.0}, {'current': 1772.758, 'min': 1500.0, 'max': 2250.0}, {'current': 1738.087, 'min': 1500.0, 'max': 2250.0}, {'current': 2311.378, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.599, 'min': 1500.0, 'max': 2250.0}, {'current': 1620.967, 'min': 1500.0, 'max': 2250.0}, {'current': 1864.863, 'min': 1500.0, 'max': 2250.0}, {'current': 1729.177, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.804, 'min': 1500.0, 'max': 2250.0}, {'current': 1788.133, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.255, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.306, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.529, 'min': 1500.0, 'max': 2250.0}, {'current': 1733.742, 'min': 1500.0, 'max': 2250.0}, {'current': 2473.813, 'min': 1500.0, 'max': 2250.0}, {'current': 1932.309, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.805, 'min': 1500.0, 'max': 2250.0}, {'current': 1730.935, 'min': 1500.0, 'max': 2250.0}, {'current': 1729.65, 'min': 1500.0, 'max': 2250.0}, {'current': 1730.989, 'min': 1500.0, 'max': 2250.0}, {'current': 2002.554, 'min': 1500.0, 'max': 2250.0}, {'current': 2089.927, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.843, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.434, 'min': 1500.0, 'max': 2250.0}, {'current': 2271.01, 'min': 1500.0, 'max': 2250.0}, {'current': 3370.134, 'min': 1500.0, 'max': 2250.0}, {'current': 1687.104, 'min': 1500.0, 'max': 2250.0}, {'current': 1688.345, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.029, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.66, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.621, 'min': 1500.0, 'max': 2250.0}, {'current': 1686.668, 'min': 1500.0, 'max': 2250.0}, {'current': 1687.585, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.907, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.023, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.835, 'min': 1500.0, 'max': 2250.0}, {'current': 2090.043, 'min': 1500.0, 'max': 2250.0}, {'current': 2245.903, 'min': 1500.0, 'max': 2250.0}, {'current': 2202.881, 'min': 1500.0, 'max': 2250.0}, {'current': 2284.838, 'min': 1500.0, 'max': 2250.0}, {'current': 1598.829, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.082, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.638, 'min': 1500.0, 'max': 2250.0}, {'current': 1649.714, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.626, 'min': 1500.0, 'max': 2250.0}, {'current': 3027.152, 'min': 1500.0, 'max': 2250.0}, {'current': 2017.291, 'min': 1500.0, 'max': 2250.0}, {'current': 2012.466, 'min': 1500.0, 'max': 2250.0}, {'current': 1990.236, 'min': 1500.0, 'max': 2250.0}, {'current': 2114.394, 'min': 1500.0, 'max': 2250.0}, {'current': 2095.433, 'min': 1500.0, 'max': 2250.0}, {'current': 2096.911, 'min': 1500.0, 'max': 2250.0}, {'current': 1840.403, 'min': 1500.0, 'max': 2250.0}, {'current': 2203.59, 'min': 1500.0, 'max': 2250.0}, {'current': 2185.592, 'min': 1500.0, 'max': 2250.0}, {'current': 2201.94, 'min': 1500.0, 'max': 2250.0}, {'current': 2018.775, 'min': 1500.0, 'max': 2250.0}, {'current': 2200.874, 'min': 1500.0, 'max': 2250.0}, {'current': 1873.351, 'min': 1500.0, 'max': 2250.0}, {'current': 3369.243, 'min': 1500.0, 'max': 2250.0}, {'current': 2235.133, 'min': 1500.0, 'max': 2250.0}, {'current': 3367.352, 'min': 1500.0, 'max': 2250.0}, {'current': 3364.254, 'min': 1500.0, 'max': 2250.0}, {'current': 2202.453, 'min': 1500.0, 'max': 2250.0}, {'current': 2038.768, 'min': 1500.0, 'max': 2250.0}, {'current': 2233.914, 'min': 1500.0, 'max': 2250.0}, {'current': 2182.285, 'min': 1500.0, 'max': 2250.0}, {'current': 2302.527, 'min': 1500.0, 'max': 2250.0}, {'current': 2185.247, 'min': 1500.0, 'max': 2250.0}, {'current': 2132.843, 'min': 1500.0, 'max': 2250.0}, {'current': 2971.398, 'min': 1500.0, 'max': 2250.0}, {'current': 2055.752, 'min': 1500.0, 'max': 2250.0}, {'current': 1567.353, 'min': 1500.0, 'max': 2250.0}, {'current': 1609.825, 'min': 1500.0, 'max': 2250.0}, {'current': 1606.242, 'min': 1500.0, 'max': 2250.0}, {'current': 1610.589, 'min': 1500.0, 'max': 2250.0}, {'current': 1617.499, 'min': 1500.0, 'max': 2250.0}, {'current': 2398.668, 'min': 1500.0, 'max': 2250.0}, {'current': 3020.272, 'min': 1500.0, 'max': 2250.0}, {'current': 2053.074, 'min': 1500.0, 'max': 2250.0}, {'current': 2141.083, 'min': 1500.0, 'max': 2250.0}, {'current': 2182.762, 'min': 1500.0, 'max': 2250.0}, {'current': 2190.047, 'min': 1500.0, 'max': 2250.0}, {'current': 2209.589, 'min': 1500.0, 'max': 2250.0}, {'current': 2072.985, 'min': 1500.0, 'max': 2250.0}, {'current': 2427.726, 'min': 1500.0, 'max': 2250.0}, {'current': 2205.137, 'min': 1500.0, 'max': 2250.0}, {'current': 2197.942, 'min': 1500.0, 'max': 2250.0}, {'current': 2208.468, 'min': 1500.0, 'max': 2250.0}, {'current': 3307.897, 'min': 1500.0, 'max': 2250.0}, {'current': 2210.539, 'min': 1500.0, 'max': 2250.0}, {'current': 2580.208, 'min': 1500.0, 'max': 2250.0}, {'current': 2487.416, 'min': 1500.0, 'max': 2250.0}, {'current': 3360.967, 'min': 1500.0, 'max': 2250.0}, {'current': 2409.573, 'min': 1500.0, 'max': 2250.0}, {'current': 2399.184, 'min': 1500.0, 'max': 2250.0}, {'current': 2400.551, 'min': 1500.0, 'max': 2250.0}, {'current': 2453.642, 'min': 1500.0, 'max': 2250.0}, {'current': 1814.426, 'min': 1500.0, 'max': 2250.0}, {'current': 2440.557, 'min': 1500.0, 'max': 2250.0}, {'current': 2422.632, 'min': 1500.0, 'max': 2250.0}, {'current': 1919.862, 'min': 1500.0, 'max': 2250.0}, {'current': 2519.195, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.938, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.157, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.328, 'min': 1500.0, 'max': 2250.0}, {'current': 1797.233, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.218, 'min': 1500.0, 'max': 2250.0}, {'current': 1790.531, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1614.0720481872559}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-03-27 19:04:19,108 INFO HandlerThread:1893870 [system_monitor.py:probe():224] Finished collecting system info +2024-03-27 19:04:19,108 INFO HandlerThread:1893870 [system_monitor.py:probe():227] Publishing system info +2024-03-27 19:04:19,109 INFO HandlerThread:1893870 [system_monitor.py:probe():229] Finished publishing system info +2024-03-27 19:04:19,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:19,113 DEBUG SenderThread:1893870 [sender.py:send():382] send: files +2024-03-27 19:04:19,113 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-27 19:04:19,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: python_packages +2024-03-27 19:04:19,118 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: python_packages +2024-03-27 19:04:19,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:04:19,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:04:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:04:19,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:19,280 DEBUG SenderThread:1893870 [sender.py:send():382] send: telemetry +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: config +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: telemetry +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:04:19,281 WARNING SenderThread:1893870 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: telemetry +2024-03-27 19:04:19,625 INFO wandb-upload_0:1893870 [upload_job.py:push():131] Uploaded file /tmp/tmp4zjika_rwandb/yd0vpcfo-wandb-metadata.json +2024-03-27 19:04:19,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-metadata.json +2024-03-27 19:04:19,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/requirements.txt +2024-03-27 19:04:19,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:21,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:21,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:21,990 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:23,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:23,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:24,283 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:26,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:26,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:28,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:28,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:29,285 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:31,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:31,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:33,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:33,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:33,993 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:34,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:04:34,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:04:34,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:04:34,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:35,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:35,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:37,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:37,751 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:37,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:39,995 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:40,267 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:40,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:40,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:42,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:42,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:43,996 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:45,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:45,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:45,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:45,997 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:47,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:47,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:49,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:04:49,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:04:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:04:49,998 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:50,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:50,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:50,804 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:50,999 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/config.yaml +2024-03-27 19:04:51,999 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:52,541 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:52,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:56,001 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:56,107 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:04:56,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:56,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:56,561 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:58,002 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:58,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:58,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:00,002 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:00,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:00,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:02,026 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:02,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:02,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:04,003 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:04,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:04,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:04,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:05,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:05,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:06,004 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:07,567 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:07,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:07,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:09,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:09,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:10,005 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:11,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:11,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:12,005 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:13,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:14,006 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:14,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:14,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:16,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:16,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:18,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:18,608 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:19,057 DEBUG SystemMonitor:1893870 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-27 19:05:19,059 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:05:19,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:19,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:19,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:19,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:20,008 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:21,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:21,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:23,724 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:24,009 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:24,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:24,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:26,009 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:26,712 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:26,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:28,010 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:29,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:29,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:29,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:33,689 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:05:33,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:33,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:34,011 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:34,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:34,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:34,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:34,300 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:36,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:36,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:38,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:38,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:40,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:40,219 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:41,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:41,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:43,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:43,751 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:44,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:46,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:46,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:46,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:48,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:48,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:49,061 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:05:49,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:49,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:50,015 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:50,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:50,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:51,270 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:53,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:53,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:55,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:55,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:56,016 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:57,202 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:58,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:58,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:00,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:00,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:01,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:06:01,277 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:06:01,279 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:06:02,017 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:02,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:06:02,281 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:03,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:03,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:04,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:04,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:04,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:04,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:05,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:05,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:08,019 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:08,224 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:08,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:08,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:12,549 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:06:12,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:12,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:13,225 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:14,021 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:15,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:15,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:17,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:17,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:18,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:19,063 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:06:19,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:19,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:20,022 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:20,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:20,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:22,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:22,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:23,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:25,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:25,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:26,023 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:27,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:27,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:28,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:29,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/config.yaml +2024-03-27 19:06:30,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:30,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:32,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:32,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:32,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:34,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:34,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:34,262 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:35,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:35,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:37,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:37,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:38,026 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:39,427 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:40,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:40,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:42,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:42,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:44,028 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:44,460 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:45,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:45,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:47,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:47,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:49,064 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:06:49,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:49,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:49,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:51,181 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:06:51,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:51,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:52,029 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:54,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:54,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:54,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:56,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:56,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:58,031 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:59,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:59,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:59,707 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:01,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:01,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:04,032 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:04,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:04,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:04,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:04,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:04,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:05,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:06,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:06,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:09,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:09,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:10,034 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:10,797 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:11,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:11,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:14,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:14,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:15,851 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:16,035 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:16,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:16,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:19,066 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:07:19,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:19,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:19,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:19,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:20,900 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:21,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:21,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:22,037 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:24,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:24,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:25,901 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:26,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:26,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:28,038 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:30,090 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:07:30,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:30,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:30,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:32,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:32,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:34,040 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:34,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:34,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:34,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:34,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:36,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:37,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:37,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:39,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:39,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:40,042 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:41,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:42,051 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:42,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:44,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:44,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:46,043 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:47,105 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:48,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:48,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:49,068 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:07:49,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:49,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:49,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:50,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:50,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:52,045 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:52,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:53,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:53,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:55,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:55,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:57,227 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:58,047 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:58,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:58,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:00,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:00,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:02,227 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:03,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:03,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:04,049 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:04,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:04,121 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:05,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:05,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:07,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:08,731 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:08:08,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:08,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:10,051 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:11,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:11,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:13,318 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:13,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:13,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:16,052 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:16,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:16,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:18,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:18,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:18,613 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:19,069 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:08:19,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:19,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:19,121 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:21,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:21,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:22,054 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:23,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:23,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:23,687 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:26,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:26,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:28,056 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:28,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:28,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:28,739 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:31,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:31,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:32,869 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:08:32,870 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:08:32,870 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:08:32,871 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:08:33,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:08:33,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:33,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:33,871 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:34,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:34,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:34,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:36,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:36,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:38,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:38,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:39,862 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:40,060 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:41,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:41,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:43,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:43,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:44,863 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:46,062 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:46,988 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:08:47,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:47,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:49,071 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:08:49,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:49,121 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:50,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:50,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:50,287 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:52,064 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:52,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:52,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:54,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:54,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:55,968 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:56,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:56,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:58,066 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:58,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:58,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:01,030 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:01,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:01,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:03,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:03,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:04,068 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:04,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:06,347 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:06,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:06,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:08,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:08,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:10,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:11,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:11,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:12,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:13,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:13,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:15,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:15,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:16,071 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:17,205 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:18,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:18,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:19,073 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:09:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:19,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:20,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:20,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:22,075 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:22,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:24,380 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:09:24,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:24,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:27,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:27,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:27,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:28,077 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:29,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:29,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:32,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:32,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:32,477 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:34,079 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:34,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:34,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:34,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:37,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:37,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:38,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:39,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:39,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:40,081 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:42,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:42,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:43,588 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:44,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:44,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:46,083 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:47,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:47,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:48,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:49,075 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:09:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:49,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:49,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:49,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:52,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:52,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:53,710 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:54,085 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:54,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:54,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:56,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:56,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:58,771 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:59,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:59,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:00,087 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:03,234 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:10:03,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:03,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:03,772 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:04,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:06,089 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:06,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:06,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:08,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:08,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:09,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:10,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:10,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:12,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:13,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:13,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:14,883 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:15,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:15,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,094 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:18,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:19,076 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:10:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:19,121 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:19,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:20,241 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:20,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:20,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:23,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:23,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:24,096 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:25,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:25,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:25,375 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:27,309 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:27,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,098 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:30,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,433 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:32,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:32,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:34,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:34,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:34,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:35,479 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:36,100 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:37,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:37,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:40,480 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:40,798 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:10:40,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:40,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:42,102 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:42,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:42,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:45,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:45,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:45,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:47,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:47,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:48,104 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:49,078 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:10:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:49,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:49,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:49,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:49,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:51,234 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:52,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:52,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:54,107 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:54,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:54,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:56,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:56,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:56,775 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:59,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:59,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:00,109 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:01,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:01,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:01,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:04,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:04,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:04,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:04,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:05,256 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:11:05,258 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:11:05,258 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:11:05,258 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:11:06,111 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:11:06,111 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:06,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:06,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:07,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:08,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:08,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:11,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:11,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:12,113 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:12,502 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:13,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:13,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:16,902 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:11:16,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:16,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:17,503 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:18,115 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:19,080 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:11:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:19,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:19,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:19,771 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:21,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:21,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:23,319 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:24,117 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:24,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:24,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:26,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:26,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:28,626 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:29,865 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:29,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:30,119 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:31,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:31,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:33,986 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:34,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:34,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:34,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:36,121 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:36,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:36,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:39,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:39,038 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:39,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:40,122 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:41,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:41,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,109 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:44,124 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:46,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:46,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:46,124 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:48,125 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:48,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:48,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:49,080 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:11:49,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:49,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:50,800 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:51,028 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:51,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:52,126 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:54,631 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:11:54,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:54,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:55,880 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:57,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:57,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:58,129 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:59,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:59,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:01,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:01,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:01,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:04,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:04,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:04,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:04,131 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:04,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:04,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:06,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:06,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:07,276 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:08,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:08,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:10,133 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:11,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:11,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:12,629 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:13,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:13,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:16,135 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:16,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:16,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:17,689 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:18,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:18,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:19,084 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:12:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:19,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:21,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:21,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:22,137 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:22,969 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:23,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:23,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:26,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:26,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:28,019 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:28,139 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:28,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:28,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:33,004 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:12:33,020 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:33,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:33,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:34,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:34,141 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:35,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:35,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:37,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:37,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:38,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:40,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:40,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:40,144 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:42,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:42,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:44,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:44,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:44,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:46,146 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:46,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:46,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:49,084 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:12:49,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:49,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:49,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:49,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:49,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:50,941 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:51,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:51,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:52,148 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:55,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:55,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:56,775 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:57,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:57,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:58,150 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:59,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:59,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:01,834 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:02,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:02,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:04,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:04,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:04,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:04,152 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:05,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:05,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:07,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:07,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:07,293 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:10,682 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:13:10,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:10,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:12,155 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:12,961 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:13,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:13,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:15,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:15,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:17,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:17,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:18,009 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:18,157 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:19,085 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:13:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:19,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:20,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:20,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:22,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:13:22,132 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:13:22,133 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:13:22,135 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:13:22,159 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:13:22,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:22,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:23,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:24,159 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:25,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:25,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:27,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:27,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:29,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:30,161 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:30,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:30,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:32,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:32,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:34,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:34,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:35,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:35,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:36,163 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:37,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:37,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:39,347 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:40,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:40,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:42,166 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:42,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:42,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:44,465 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:45,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:45,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:48,168 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:49,087 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:13:49,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:49,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:49,411 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:13:49,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:49,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:50,264 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:52,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:52,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:54,170 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:54,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:54,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:55,582 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:57,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:57,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:59,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:59,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:00,172 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:00,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:02,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:02,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:04,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:04,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:04,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:04,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:04,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:05,698 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:06,174 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:07,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:07,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:09,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:09,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:10,699 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:12,175 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:12,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:12,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:14,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:14,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:15,772 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:17,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:17,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:18,178 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:19,089 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:14:19,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:19,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:19,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:19,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:21,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:22,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:22,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:24,180 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:24,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:24,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:26,888 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:28,076 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:14:28,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:28,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:30,182 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:30,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:30,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:31,936 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:32,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:32,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:34,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:34,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:35,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:35,837 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:36,184 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:36,988 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:37,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:37,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:40,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:40,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:42,056 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:42,186 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:42,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:42,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:45,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:45,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:47,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:47,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:47,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:48,188 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:49,091 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:14:49,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:49,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:50,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:51,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:52,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:53,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:53,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:54,190 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:56,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:56,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:00,192 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:00,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:00,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,244 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:04,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:04,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:04,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:06,423 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:15:06,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:06,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:07,195 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:08,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:09,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:09,152 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:11,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:11,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:13,197 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:13,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:14,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:14,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:16,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:16,217 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:18,428 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:19,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:19,092 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:15:19,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:19,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:19,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:19,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:21,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:21,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:23,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:24,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:24,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:25,201 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:26,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:26,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:28,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:28,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:28,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:31,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:31,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:31,203 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:33,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:33,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:34,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:34,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:36,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:36,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:37,206 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:38,613 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:39,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:39,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:41,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:41,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:43,209 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:43,676 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:44,609 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:15:44,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:44,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:47,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:47,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:48,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:49,093 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:15:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:49,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:49,212 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:49,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:49,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:52,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:52,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:53,878 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:53,950 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:15:53,951 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:15:53,952 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:15:53,952 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:15:54,214 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:15:54,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:54,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:55,214 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:56,220 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:57,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:57,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:58,953 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:59,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:59,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:01,222 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:02,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:02,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:04,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:04,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:04,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:04,246 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:04,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:04,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:06,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:06,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:07,225 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:09,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:09,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:10,013 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:11,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:11,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:13,228 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:14,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:14,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:15,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:16,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:16,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:19,095 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:16:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:19,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:19,230 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:19,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:19,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:20,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:23,037 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:16:23,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:23,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:27,233 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:28,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:28,203 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:31,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:31,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:31,245 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:33,235 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:33,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:33,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:34,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:34,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:36,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:36,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:36,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:38,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:38,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:39,243 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:41,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:41,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:41,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:43,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:43,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:45,245 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:46,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:46,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:46,367 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:48,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:48,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:49,096 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:16:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:49,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:51,247 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:51,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:51,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:52,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:53,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:53,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:56,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:56,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:57,250 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:57,478 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:58,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:58,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:01,917 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:17:01,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:01,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:02,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:03,252 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:04,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:04,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:04,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:04,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:06,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:06,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:07,621 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:09,255 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:09,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:09,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:11,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:11,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:12,622 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:14,961 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:14,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:15,257 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:16,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:17,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:17,678 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:19,098 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:17:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:19,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:19,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:19,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:21,259 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:21,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:21,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:23,356 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:24,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:24,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:26,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:27,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:27,261 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:28,831 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:29,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:29,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:31,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:32,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:33,265 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:33,893 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:34,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:34,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:34,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:34,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:37,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:37,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:38,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:39,267 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:40,552 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:17:40,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:40,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:43,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:43,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:44,009 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:45,269 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:45,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:45,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:48,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:48,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:49,100 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:17:49,101 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:49,271 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:50,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:50,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:53,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:53,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:54,331 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:55,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:55,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:57,273 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:58,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:58,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:59,331 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:00,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:00,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:01,274 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:03,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:03,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:04,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:04,341 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:05,276 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:05,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:05,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:07,276 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:08,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:08,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:10,341 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:10,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:10,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:11,099 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:18:11,100 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:18:11,101 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:18:11,102 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:18:11,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:18:11,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:12,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:13,279 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:13,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:13,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:15,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:15,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:16,103 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:19,101 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:18:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:19,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:19,248 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:18:19,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:19,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:19,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:21,280 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:22,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:22,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:23,287 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:24,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:24,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:27,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:27,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:27,247 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:29,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:29,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:29,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:32,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:32,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:32,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:34,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:34,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:34,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:36,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:36,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:37,291 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:37,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:38,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:38,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:41,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:41,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:42,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:43,293 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:44,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:44,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:47,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:47,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:47,508 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:49,102 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:18:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:49,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:49,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:49,295 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:52,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:52,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:52,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:54,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:54,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:55,297 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:57,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:57,754 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:18:57,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:57,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:00,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:00,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:01,299 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:02,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:02,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:02,612 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:04,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:05,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:05,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:07,301 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:07,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:07,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:07,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:10,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:10,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,731 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:13,303 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:15,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:15,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:17,800 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:17,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:17,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:19,104 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:19:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:19,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:19,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:19,305 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:20,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:20,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:22,854 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:22,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:22,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:24,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:24,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:25,306 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:27,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:27,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:27,855 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:29,865 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:29,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:31,308 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:31,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:31,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:32,913 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:34,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:35,284 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:19:35,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:35,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:37,310 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:37,990 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:38,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:38,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:40,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:40,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:42,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:42,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:43,034 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:43,312 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:44,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:44,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:47,001 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:47,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:48,087 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:49,106 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:19:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:49,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:49,314 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:50,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:50,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:52,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:52,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:53,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:55,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:55,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:55,315 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:57,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:57,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:58,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:00,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:00,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:01,317 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:02,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:02,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:03,220 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:04,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:04,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:04,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:06,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:06,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:07,323 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:08,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:08,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:08,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:12,526 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:20:12,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:12,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:13,325 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:13,340 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:15,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:15,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:17,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:17,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:18,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:19,107 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:20:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:19,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:19,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:19,327 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:20,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:20,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:22,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:22,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:23,464 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:25,330 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:25,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:25,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:27,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:27,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:28,464 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:30,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:30,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:31,332 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:32,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:32,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:33,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:34,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:35,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:35,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:37,334 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:37,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:37,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:38,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:40,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:40,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:41,719 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:20:41,720 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:20:41,721 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:20:41,721 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:20:42,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:20:42,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:42,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:42,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:43,336 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:43,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:45,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:45,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:47,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:47,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:49,108 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:20:49,109 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:49,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:49,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:49,339 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:51,501 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:20:51,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:51,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:54,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:54,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:54,766 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:55,341 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:56,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:56,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:58,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:58,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:59,767 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:01,342 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:01,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:01,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:03,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:03,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:04,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:05,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:06,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:06,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:07,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:08,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:08,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:10,888 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:11,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:11,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:13,347 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:13,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:13,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:15,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:16,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:16,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:18,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:18,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:19,110 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:21:19,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:19,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:19,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:19,349 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:21,334 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:21,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:21,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:23,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:23,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:25,351 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:26,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:26,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:27,060 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:30,537 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:21:30,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:30,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:31,353 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:32,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:33,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:33,595 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:34,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:35,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:35,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:37,199 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:37,355 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:38,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:38,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:40,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:40,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:42,200 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:43,357 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:43,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:43,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:45,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:45,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:47,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:48,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:48,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:49,111 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:21:49,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:49,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:49,359 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:50,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:50,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:52,326 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:53,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:53,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:55,361 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:55,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:55,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:57,385 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:57,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:57,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:59,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:00,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:01,363 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:02,452 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:03,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:03,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:04,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:05,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:05,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:07,366 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:07,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:08,705 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:22:08,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:08,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:11,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:11,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:12,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:13,368 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:13,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:13,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:16,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:16,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:17,597 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:18,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:18,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:19,113 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:22:19,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:19,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:20,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:20,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:21,371 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:23,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:24,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:24,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:26,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:26,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:27,373 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:28,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:28,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:28,740 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:31,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:31,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:33,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:33,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:33,375 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:33,811 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:34,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:36,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:36,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:38,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:38,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:38,865 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:39,377 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:41,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:41,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:43,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:43,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:43,955 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:45,379 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:47,324 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:22:47,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:47,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:48,955 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:49,115 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:22:49,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:49,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:49,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:49,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:51,381 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:51,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:51,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:54,254 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:54,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:54,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:56,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:56,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:57,383 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:59,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:59,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:00,098 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:01,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:01,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:03,385 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:04,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:04,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:04,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:04,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:05,256 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:06,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:06,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:08,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:08,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:09,387 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:10,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:10,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:11,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:12,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:12,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:13,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:23:13,408 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:23:13,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:23:13,410 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:23:14,389 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:23:15,390 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:15,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:15,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:16,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:17,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:17,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:19,116 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:23:19,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:19,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:20,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:20,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:21,392 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:21,432 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:24,408 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:23:24,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:24,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:26,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:27,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:27,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:27,394 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:29,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:29,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:31,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:31,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:31,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:33,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:33,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:33,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:34,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:34,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:36,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:36,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:37,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:38,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:38,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:39,399 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:41,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:41,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:42,690 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:43,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:43,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:45,401 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:46,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:46,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:47,784 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:48,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:48,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:49,118 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:23:49,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:49,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:51,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:51,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:51,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:53,674 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:53,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:53,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:57,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:57,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:57,405 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:58,953 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:59,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:59,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:02,540 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:24:02,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:02,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:03,408 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:04,032 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:04,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:04,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:05,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:05,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:07,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:07,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:09,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:09,410 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:10,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:10,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:12,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:12,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:14,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:15,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:15,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:15,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:17,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:17,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:19,119 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:24:19,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:19,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:19,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:20,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:20,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:22,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:22,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:23,416 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:24,797 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:25,417 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:25,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:25,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:27,417 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:27,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:27,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:29,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:29,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:30,300 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:31,419 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:32,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:32,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:33,420 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:34,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:34,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:34,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:34,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:34,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:36,266 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:36,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:36,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:37,421 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:39,941 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:24:39,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:39,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:41,764 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:42,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:42,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:43,424 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:44,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:44,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:47,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:47,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:47,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:49,121 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:24:49,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:49,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:49,426 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:49,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:49,879 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:52,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:52,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:52,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:54,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:54,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:55,429 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:57,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:57,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:57,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:59,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:59,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:01,430 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:02,671 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:03,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:03,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:04,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:04,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:04,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:05,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:05,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:07,433 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:07,743 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:08,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:08,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:10,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:10,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:13,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,435 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:15,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:15,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:18,067 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:18,626 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:25:18,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:18,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:19,122 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:25:19,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:19,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:19,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:21,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:21,073 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:23,090 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:23,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:23,330 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:25,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:25,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:27,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:27,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:29,079 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:29,441 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:30,961 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:30,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:32,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:33,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:33,442 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:34,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:34,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:34,315 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:35,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:25:35,106 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:25:35,106 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:25:35,106 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:25:35,443 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:25:36,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:36,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:37,444 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:38,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:38,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:40,107 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:40,940 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:40,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:42,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:42,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:43,446 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:45,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:45,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:46,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:47,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:47,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:49,125 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:25:49,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:49,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:49,448 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:49,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:49,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:51,296 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:52,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:52,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:55,450 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:56,348 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:25:56,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:56,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:57,229 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:58,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:58,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:01,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:01,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:01,453 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:02,297 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:03,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:03,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:04,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:04,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:04,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:06,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:06,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:07,381 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:07,455 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:08,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:08,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:11,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:11,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:12,456 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:13,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:13,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:13,459 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:16,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:16,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:17,457 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:18,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:18,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:19,125 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:26:19,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:19,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:19,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:19,466 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:20,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:20,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:25,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:25,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:25,469 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:27,234 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:27,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:28,898 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:29,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:30,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:31,471 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:33,730 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:26:33,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:33,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:33,975 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:34,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:34,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:35,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:35,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:37,474 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:38,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:38,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:39,030 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:40,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:40,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:43,476 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:43,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:43,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:44,097 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:45,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:45,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:48,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:48,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:49,128 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:26:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:49,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:49,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:49,478 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:50,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:50,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:53,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:53,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:54,227 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:55,480 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:55,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:55,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:57,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:57,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:59,228 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:00,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:00,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:01,482 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:02,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:02,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:04,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:04,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:04,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:04,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:05,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:05,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:07,485 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:07,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:07,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:09,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:11,255 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:27:11,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:11,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:13,487 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:14,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:14,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:14,615 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:16,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:16,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:19,130 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:27:19,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:19,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:19,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,489 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:20,265 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:21,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:21,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:23,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:23,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:25,491 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:25,747 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:26,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:26,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:28,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:28,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:30,748 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:31,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:31,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:31,493 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:33,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:33,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:34,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:34,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:36,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:36,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:36,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:37,495 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:38,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:38,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:41,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:41,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:41,879 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:43,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:43,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:43,497 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:46,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:46,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:46,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:49,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:49,258 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:27:49,698 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:27:49,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:49,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:51,500 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:52,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:52,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:52,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:54,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:54,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:57,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:57,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:57,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:58,221 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:59,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:59,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:02,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:02,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:03,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:03,505 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:04,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:04,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:04,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:04,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:07,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:28:07,418 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:28:07,419 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:28:07,421 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:28:07,506 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:28:07,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:07,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:08,422 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:09,507 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:09,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:09,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:12,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:12,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:14,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:15,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:15,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:15,510 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:18,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:18,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:19,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:19,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:19,348 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:28:19,490 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:20,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:20,090 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:21,512 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:22,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:22,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:24,490 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:24,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:25,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:27,515 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:28,533 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:28:28,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:28,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:29,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:31,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:31,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:33,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:33,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:33,518 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:34,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:34,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:35,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:35,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:35,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:38,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:38,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:39,520 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:40,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:40,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:40,674 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:43,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:43,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,522 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:45,732 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:48,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:48,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:49,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:49,315 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:28:50,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:50,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:50,886 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:51,524 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:53,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:53,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,886 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:57,526 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:58,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:58,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:00,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:00,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:00,955 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:03,529 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:03,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:03,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:04,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:04,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:06,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:07,139 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:29:07,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:07,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:09,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:09,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:09,531 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:12,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:12,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:12,085 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:14,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:14,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:15,533 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:17,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:17,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:17,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:19,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:19,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:19,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:19,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:20,751 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:29:21,535 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:22,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:22,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:22,752 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:24,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:24,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:26,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:26,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:27,537 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:28,278 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:29,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:29,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:31,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:31,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:33,338 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:33,539 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:34,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:34,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:34,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:34,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:36,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:36,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:38,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:39,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:39,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:39,542 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:41,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:41,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:43,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:44,943 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:29:44,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:44,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:45,545 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:47,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:47,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:48,468 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:49,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:49,265 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:29:49,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:49,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:51,547 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:52,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:52,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:53,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:54,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:54,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:57,549 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:57,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:57,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:58,576 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:59,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:59,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:01,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:01,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:03,551 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:03,645 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:04,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:04,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:04,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:04,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:06,757 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:06,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:08,694 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:09,554 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:09,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:09,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:11,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:11,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:13,695 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:14,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:14,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:15,555 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:16,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:16,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:18,759 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:19,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:19,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:19,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:19,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:20,265 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:30:21,558 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:23,396 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:30:23,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:23,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:23,832 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:26,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:26,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:27,560 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:28,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:28,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:28,877 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:30,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:30,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:33,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:33,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:33,562 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:33,932 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:34,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:34,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:35,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:35,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:38,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:38,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:38,981 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:39,052 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:30:39,054 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:30:39,054 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:30:39,054 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:30:39,565 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:30:39,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:40,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:40,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:41,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:43,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:43,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:44,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:45,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:45,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:45,568 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:48,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:48,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:49,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:49,575 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:50,322 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:30:50,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:50,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:50,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:51,575 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:53,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:53,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,577 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:55,839 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:57,577 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:58,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:58,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:01,423 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:02,145 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:31:02,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:02,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:03,579 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:04,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:04,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:04,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:04,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:06,725 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:06,940 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:06,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:07,580 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:08,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:08,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:11,725 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:11,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:11,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:13,582 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:13,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:13,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:16,794 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:16,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:16,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:18,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:18,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:19,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:19,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:19,347 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:31:19,584 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:21,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:21,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:22,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:23,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:23,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:25,586 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:25,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:25,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:27,907 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:28,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:28,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,588 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:32,981 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:34,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:34,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:34,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:36,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:36,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:37,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:38,257 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:39,657 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:31:39,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:39,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:42,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:42,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:43,257 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:44,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:44,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:45,592 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:47,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:47,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:48,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:49,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:49,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:49,307 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:31:49,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:49,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:51,594 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:52,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:52,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:53,381 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:54,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:54,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,596 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:58,436 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:59,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:59,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:02,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:02,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:03,481 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:03,598 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:04,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:04,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:04,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:04,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:04,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:06,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:06,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:08,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:09,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:09,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:09,600 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:11,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:11,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:13,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:14,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:14,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:15,603 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:17,983 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:32:18,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:18,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:18,602 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:19,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:19,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:19,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:20,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:20,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:20,887 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:32:21,605 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:22,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:22,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:23,888 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:24,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:24,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:27,607 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:27,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:27,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:29,738 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:29,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:29,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:31,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:31,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:33,609 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:34,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:34,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:34,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:34,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:35,349 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:36,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:36,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,611 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:40,860 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:41,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:41,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:44,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:44,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:45,613 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:45,903 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:46,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:46,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:49,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:49,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:49,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:49,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:50,512 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:32:51,514 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:51,615 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:51,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:51,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:55,145 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:32:55,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:55,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:56,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:32:56,106 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:32:56,106 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:32:56,107 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:32:56,616 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:32:57,107 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:57,617 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:58,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:58,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:00,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:00,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:02,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:02,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:03,088 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:03,618 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:04,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:04,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:04,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:05,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:05,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:08,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:08,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:08,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:09,620 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:10,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:10,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:13,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:13,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:13,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,622 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:18,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:18,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:18,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:19,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:19,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:19,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:20,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:20,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:20,580 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:33:21,624 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:23,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:23,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:23,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:25,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:25,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:27,626 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:28,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:28,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:29,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:30,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:30,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:33,788 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:33:33,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:33,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:34,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:34,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:34,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:34,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:34,628 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:36,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:36,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:38,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:38,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:39,424 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:40,630 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:41,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:41,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:43,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:43,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:44,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:46,632 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:46,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:46,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:48,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:48,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:49,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:49,351 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:33:50,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:50,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:51,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:52,634 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:53,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:53,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:55,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:56,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:56,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:58,636 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:58,777 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:58,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:00,594 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:01,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:01,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:03,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:03,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:04,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:04,132 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:04,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:04,638 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:06,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:06,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:06,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:08,639 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:08,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:08,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:10,640 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:11,870 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:12,291 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:34:12,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:12,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:14,641 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:15,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:15,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:16,642 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:17,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:17,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:17,601 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:19,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:19,132 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:19,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:19,289 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:34:20,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:20,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:20,643 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:22,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:22,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:23,292 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:25,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:25,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:26,645 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:27,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:27,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:28,438 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:30,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:30,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:30,647 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:32,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:32,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:34,010 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:34,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:34,132 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:34,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:34,648 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:35,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:35,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:37,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:37,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:39,917 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:40,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:40,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:40,650 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:42,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:42,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:45,056 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:45,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:45,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:46,652 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:47,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:47,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:49,133 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:49,310 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:34:50,335 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:50,563 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:34:50,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:50,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:50,654 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:53,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:53,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:55,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:55,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:56,244 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:56,656 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:58,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:58,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:00,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:00,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:01,894 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:02,658 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:03,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:03,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:04,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:04,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:04,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:05,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:05,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:05,847 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:35:05,848 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:35:05,849 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:35:05,849 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:35:06,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:35:07,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:07,850 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:08,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:08,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:08,661 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:10,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:10,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:13,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:13,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:13,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:14,663 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:15,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:15,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,665 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:19,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:19,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:19,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:20,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:20,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:20,455 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:35:20,455 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:22,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:22,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:24,667 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:24,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:24,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:26,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:28,199 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:35:28,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:28,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:28,669 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:31,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:31,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:32,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:33,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:33,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:34,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:34,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:34,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:34,671 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:35,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:35,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:37,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:37,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:37,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:40,673 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:40,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:40,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:42,605 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:42,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:42,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:44,674 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:45,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:45,793 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:47,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:47,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:48,584 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:49,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:49,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:50,234 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:35:50,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:50,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,676 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:54,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:55,467 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:55,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:56,678 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:57,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:57,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:59,950 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:00,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:00,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:02,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:02,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:02,680 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:04,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:04,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:04,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:05,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:05,969 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:36:05,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:06,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:08,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:08,682 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:08,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:10,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:10,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:10,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:12,683 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:13,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:13,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:15,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:15,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:15,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:18,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:18,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:18,685 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:19,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:19,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:19,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:20,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:20,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:20,923 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:36:20,926 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:23,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:23,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:24,687 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:25,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:25,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:26,388 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:28,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,689 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:30,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:30,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:32,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:33,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:33,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:34,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:34,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:34,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:34,691 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:35,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:35,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:37,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:38,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:38,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,694 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:43,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:43,616 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:36:43,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:43,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:46,280 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:46,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:46,696 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:48,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:48,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:48,813 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:49,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:49,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:49,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:36:51,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:51,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:52,698 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:53,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:53,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:54,463 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:55,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:55,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:56,699 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:58,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:58,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:59,464 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:01,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:01,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:02,702 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:03,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:03,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:04,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:04,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:04,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:05,285 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:05,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:05,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:07,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:07,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:08,704 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:10,819 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:10,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:10,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:12,705 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:12,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:12,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:15,693 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:15,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:16,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:17,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:17,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:18,707 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:19,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:19,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:19,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:19,290 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:37:21,071 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:37:21,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:21,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:21,291 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:22,107 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:37:22,108 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:37:22,109 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:37:22,111 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:37:22,709 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:37:23,710 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:23,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:23,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:24,710 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:25,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:25,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:27,113 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:28,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:28,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:30,712 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:30,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:30,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:32,933 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:33,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:33,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:34,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:34,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:34,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:34,714 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:35,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:35,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,266 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:38,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:40,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:40,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:40,718 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:43,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:43,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:43,755 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:45,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:45,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:46,719 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:48,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:48,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:49,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:49,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:49,324 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:37:49,324 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:50,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:50,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:50,721 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:53,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:53,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:54,514 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:55,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:55,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:56,723 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:59,264 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:37:59,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:59,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:00,422 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:01,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:01,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:02,726 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:04,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:04,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:04,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:04,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:06,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:06,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:06,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:08,728 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:09,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:09,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:11,495 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:11,712 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:11,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:14,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:14,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:14,730 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:16,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:16,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:17,262 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:18,731 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:19,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:19,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:19,352 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:38:19,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:19,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:21,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:21,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:22,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:24,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:24,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:24,734 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:26,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:26,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:28,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:29,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:29,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:30,736 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:31,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:31,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:33,732 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:34,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:34,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:34,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:34,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:36,738 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:38,067 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:38:38,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:38,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:39,340 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:40,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:40,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:42,740 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:42,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:43,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:44,913 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:45,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:45,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:46,742 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:47,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:47,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:49,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:49,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:49,364 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:38:50,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:50,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:50,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:52,743 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:52,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:52,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:55,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:55,637 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:55,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:57,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:57,679 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:58,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:00,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:00,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:01,227 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:02,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:02,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:02,746 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:04,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:04,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:05,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:05,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:06,281 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:07,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:07,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:08,748 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:10,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:10,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:12,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:12,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:12,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:14,751 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:15,787 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:39:15,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:15,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:17,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:18,732 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:18,752 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:18,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:19,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:19,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:19,329 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:39:20,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:20,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:22,330 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:23,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:23,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:24,754 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:25,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:25,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:27,686 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:28,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:28,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:30,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:30,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:30,756 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:33,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:33,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:33,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:34,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:34,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:35,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:35,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:36,758 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:38,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:38,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:39,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:39,474 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:39:39,475 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:39:39,475 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:39:39,476 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:39:39,759 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:39:39,759 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:40,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:40,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:40,760 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:43,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:43,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:44,477 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:45,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:45,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:46,761 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:48,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:48,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:49,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:49,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:49,295 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:39:50,227 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:50,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:50,296 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:52,763 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:53,632 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:39:53,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:53,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:55,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:56,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,765 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:58,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:58,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:00,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:01,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:01,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:02,767 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:03,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:03,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:04,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:04,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:06,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:06,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:06,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:08,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:08,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:08,769 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:10,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:10,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:11,881 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:12,770 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:13,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:13,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:16,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:16,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:16,894 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:18,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:18,757 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:18,773 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:19,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:19,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:19,360 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:40:21,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:21,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:22,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:23,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:23,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:24,775 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:26,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:26,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:27,517 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:28,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:28,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:30,777 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:31,963 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:40:31,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:32,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:33,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:34,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:34,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:34,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:34,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:36,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:36,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:36,779 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:39,090 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:39,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:39,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:40,781 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:41,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:41,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:44,093 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:44,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:44,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:46,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:46,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:46,783 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:49,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:49,137 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:49,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:49,292 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:40:49,293 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:49,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:49,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:51,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:51,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:52,785 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:54,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:54,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:55,025 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:56,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:56,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:56,786 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:59,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:59,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:00,677 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:01,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:01,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:02,788 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:04,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:04,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:04,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:04,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:04,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:06,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:06,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:06,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:08,790 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:09,832 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:41:09,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:09,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:11,477 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:12,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:12,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,792 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:17,363 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:17,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:17,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:18,793 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:19,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:19,137 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:19,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:41:19,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:19,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:22,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:22,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:23,085 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:24,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:24,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:24,795 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:27,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:27,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:28,085 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:29,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:29,419 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:30,797 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:32,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:32,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:33,890 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:34,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:34,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:34,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:34,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:34,798 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:37,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:37,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:39,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:39,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:39,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:40,801 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:42,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:42,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:44,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:44,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:44,667 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:46,803 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:47,708 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:41:47,737 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:47,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:49,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:49,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:49,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:50,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:50,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:51,187 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:41:51,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:52,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:52,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:52,804 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:55,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:55,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:56,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:56,497 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:41:56,498 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:41:56,499 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:41:56,499 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:41:56,805 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:41:57,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:57,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:57,806 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:58,806 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:00,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:00,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:01,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:02,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:02,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:02,808 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:04,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:04,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:04,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:05,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:05,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:07,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:07,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:07,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:08,811 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:10,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:10,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:12,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:12,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:12,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:14,813 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:15,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:15,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:17,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:17,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:17,737 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:19,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:19,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:19,354 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:42:20,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:20,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:20,815 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:22,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:22,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:23,356 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:25,431 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:42:25,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:25,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:26,817 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:28,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:28,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:28,669 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:30,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:30,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:30,818 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:33,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:33,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:34,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:34,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:34,265 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:35,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:35,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:36,820 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:38,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:38,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:39,499 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:40,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:40,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:42,822 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:43,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:43,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:44,844 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:45,076 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:45,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,152 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,824 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:49,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:49,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:49,305 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:42:50,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:50,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:50,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:53,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:53,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:54,826 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:55,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:55,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:56,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:57,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:57,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:58,827 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:59,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:59,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:02,030 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:03,294 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:43:03,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:03,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:04,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:04,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:04,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:04,830 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:06,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:06,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:07,269 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:08,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:08,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:10,831 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:11,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:11,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:12,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:13,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:13,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:14,832 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:15,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:15,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:17,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:17,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:18,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:19,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:19,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:19,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:19,302 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:43:20,834 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:20,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:20,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:22,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:22,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:23,439 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:25,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:25,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:26,837 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:27,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:27,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:29,205 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:30,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:30,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:30,838 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:32,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:32,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:34,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:34,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:34,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:34,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:35,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:35,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:36,841 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:37,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:37,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:39,938 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:41,122 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:43:41,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:41,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:42,843 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:43,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:43,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:45,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:45,884 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:45,935 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:48,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:48,837 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:48,846 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:49,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:49,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:49,342 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:43:50,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:50,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:51,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:52,847 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:53,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:53,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:55,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:55,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:56,615 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:58,850 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:58,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:58,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:00,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:00,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:02,564 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:03,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:03,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:04,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:04,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:04,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:04,852 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:05,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:05,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:08,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:08,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:09,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:10,854 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:12,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:12,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:13,467 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:14,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:14,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:44:14,366 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:44:14,366 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:44:14,367 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:44:14,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:14,855 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:44:16,856 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:17,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:17,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:19,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:19,228 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:19,229 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:19,418 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:44:19,419 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:20,790 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:44:20,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:20,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:20,858 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:23,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:23,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:24,419 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:25,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:25,777 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:26,860 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:28,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:28,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:30,017 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:30,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:30,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:32,862 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:33,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:33,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:34,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:34,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:34,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:35,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:35,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:35,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:36,864 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:38,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:38,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:40,749 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:40,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:40,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:42,865 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:43,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:43,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:45,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:45,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:46,663 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:48,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:48,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:48,868 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:49,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:49,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:49,272 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:44:50,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:50,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:52,274 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:52,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:52,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:54,871 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:55,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:55,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:57,461 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:59,181 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:44:59,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:59,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:00,873 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:02,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:02,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:03,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:04,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:04,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:04,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:04,227 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:04,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:04,874 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:07,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:07,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:08,372 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:09,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:09,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:10,876 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:11,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:11,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:13,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:13,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:14,080 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:15,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:15,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:16,878 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:18,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:18,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:19,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:19,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:19,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:20,293 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:45:20,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:20,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:20,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:20,880 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:23,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:23,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:25,295 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:25,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:25,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:26,882 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:28,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:28,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:30,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:30,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:30,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:32,884 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:33,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:33,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:34,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:34,140 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:34,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:35,902 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:36,886 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:37,046 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:45:37,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:37,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:40,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:40,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:40,903 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:42,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:42,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:42,888 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:45,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:45,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:46,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:47,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:47,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:48,890 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:49,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:49,140 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:49,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:49,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:49,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:50,914 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:45:51,915 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:51,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:51,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:52,892 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:54,737 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:54,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:56,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:56,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:56,919 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:59,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:59,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:59,894 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:01,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:01,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:01,920 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:04,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:04,140 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:04,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:04,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:04,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:06,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:06,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:07,273 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:09,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:09,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:11,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:11,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:12,274 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:14,979 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:46:15,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:15,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:15,899 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:17,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:17,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:18,228 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:19,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:19,140 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:19,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:19,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:19,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:20,560 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:46:22,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:22,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:23,561 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:23,902 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:24,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:24,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:27,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:27,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:27,903 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:28,641 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:29,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:29,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:32,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:32,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:33,906 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:34,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:34,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:34,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:34,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:34,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:34,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:37,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:37,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:37,907 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:39,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:39,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:39,831 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:41,096 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:46:41,097 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:46:41,097 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:46:41,098 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:46:41,909 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:46:42,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:42,235 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:42,909 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:43,910 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:44,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:44,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:45,099 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:47,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:47,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:49,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:49,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:49,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:49,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:49,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:49,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:46:49,912 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:50,378 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:52,611 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:46:52,643 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:52,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:53,913 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:55,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:55,712 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:55,761 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:57,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:57,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,915 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:01,675 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:02,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:02,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:04,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:04,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:04,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:04,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:04,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:05,917 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:07,289 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:07,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:07,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:09,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:09,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:09,918 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:12,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:12,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:12,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:14,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:14,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:15,920 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:17,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:17,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:18,333 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:19,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:19,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:19,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:19,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:20,607 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:47:21,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:21,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:23,609 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:23,923 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:23,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:23,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,925 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:29,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:30,895 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:47:30,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:30,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:32,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:32,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:33,927 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:34,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:34,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:34,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:34,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:35,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:35,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,928 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:39,954 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:40,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:40,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:42,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:42,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:43,930 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:44,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:44,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:45,852 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:47,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:47,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:49,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:49,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:49,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:49,933 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:50,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:50,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:50,698 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:47:51,699 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:52,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:52,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:54,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:54,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:55,935 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:57,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:57,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:57,629 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:59,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:59,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:59,936 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:01,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:01,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:03,437 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:04,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:04,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:04,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:04,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:04,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:05,937 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:08,448 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:48:08,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:08,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:09,292 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:10,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:10,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:11,940 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:13,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:13,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:14,326 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:15,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:15,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:15,941 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:17,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:17,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:19,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:19,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:19,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:19,324 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:48:20,235 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:20,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:20,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:21,943 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:22,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:22,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:24,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:25,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:25,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,945 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:29,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:29,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:31,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:31,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:31,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:33,947 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:34,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:34,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:34,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:34,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:34,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:36,369 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:36,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:36,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:39,949 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:41,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:41,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:42,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:43,950 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:45,464 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:48:45,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:45,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:47,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:48,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:48,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:49,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:49,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:49,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:49,356 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:48:49,952 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:50,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:50,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:52,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:53,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:53,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:55,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:55,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:55,954 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:57,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:57,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:57,746 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:58,582 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:48:58,584 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:48:58,584 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:48:58,585 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:48:58,955 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:48:59,956 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:00,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:00,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:00,956 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:02,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:02,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:03,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:04,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:04,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:04,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:04,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:04,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:05,958 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:07,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:07,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:09,448 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:09,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:09,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:11,960 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:12,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:12,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,732 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:17,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:17,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:17,961 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:19,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:19,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:19,336 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:49:19,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:19,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:20,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:23,103 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:49:23,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:23,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:23,964 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:26,115 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:26,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:26,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:27,965 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:28,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:28,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:31,116 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:31,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:31,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:33,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:33,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:33,966 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:34,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:34,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:34,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:36,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:36,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:36,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:38,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:38,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:39,968 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:41,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:41,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:41,482 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:43,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:43,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:45,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:45,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:45,970 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:47,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:48,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:48,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:49,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:49,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:49,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:49,972 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:50,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:50,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:50,642 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:49:52,643 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:52,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:52,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:55,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:55,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:55,974 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:57,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:57,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:57,944 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:01,127 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:50:01,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:01,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:01,976 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:03,867 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:04,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:04,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:04,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:04,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:06,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:06,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:07,978 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:08,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:08,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:09,230 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:11,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:11,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:11,980 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:13,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:13,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:14,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:15,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:15,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:17,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:17,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:17,982 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:19,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:19,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:20,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:20,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:20,510 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:50:20,512 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:22,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:22,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:23,986 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:25,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:25,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:26,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:27,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:27,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:29,989 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:30,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:30,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:32,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:32,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:32,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:34,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:34,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:34,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:34,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:34,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:35,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:38,323 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:50:38,334 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:38,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:38,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:41,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:41,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:41,993 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:43,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:43,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:43,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:46,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:46,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:47,995 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:48,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:48,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:49,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:49,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:49,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:50,804 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:50:50,804 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:51,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:51,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:53,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:53,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:53,998 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:56,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:56,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:56,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:57,999 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:58,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:58,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,809 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:03,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:03,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:04,001 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:04,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:04,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:04,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:06,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:06,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:07,243 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:08,002 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:08,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:08,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:11,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:11,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:13,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:13,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:13,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:14,004 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:16,710 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:51:16,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:16,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:16,920 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:51:16,921 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:51:16,921 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:51:16,921 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:51:17,006 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:51:18,922 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:19,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:19,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:19,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:19,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:19,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:19,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:20,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:20,604 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:51:21,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:21,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:24,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:24,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:24,816 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:26,009 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:26,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:26,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:29,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:29,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:30,010 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:30,635 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:31,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:31,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:34,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:34,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:34,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:34,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:34,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:36,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:36,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:36,349 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:36,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:39,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:39,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:41,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:41,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:41,692 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:42,015 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:44,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:44,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:46,016 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:46,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:46,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:46,760 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:49,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:49,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:49,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:49,227 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:49,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:49,267 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:51:51,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:51,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:52,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:52,702 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:54,751 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:51:54,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:54,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:57,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:57,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:58,020 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:58,384 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:59,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:59,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:02,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:02,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:03,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:04,022 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:04,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:04,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:04,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:04,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:04,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:07,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:07,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,470 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:09,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:10,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:12,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:12,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:14,026 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:14,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:14,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:14,692 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:17,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:17,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:19,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:19,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:19,330 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:52:19,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:19,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:20,029 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:20,331 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:22,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:22,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:24,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:24,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:26,031 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:26,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:27,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:27,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:29,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:29,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:31,198 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:32,033 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:32,871 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:52:32,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:32,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:34,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:34,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:34,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:35,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:35,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:37,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:37,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:37,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:38,035 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:40,767 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:40,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:42,036 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:42,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:42,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:43,034 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:45,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:45,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:47,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:47,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:48,038 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:48,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:49,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:49,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:49,286 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:52:50,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:50,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:52,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:52,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:54,041 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:54,769 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:55,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:55,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:57,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:57,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:00,043 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:00,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:00,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:00,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:02,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:02,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:04,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:04,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:04,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:05,363 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:05,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:05,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:06,045 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:07,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:07,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:10,046 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:10,698 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:11,111 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:53:11,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:11,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:13,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:13,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:16,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:16,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:16,048 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:16,638 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:18,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:18,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:19,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:19,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:19,314 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:53:20,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:21,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:22,050 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:22,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:23,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:23,989 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:26,028 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:26,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:26,051 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:28,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:28,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:28,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:30,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:30,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:32,054 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:33,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:33,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:34,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:34,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:34,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:34,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:35,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:53:35,299 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:53:35,299 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:53:35,300 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:53:35,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:35,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:36,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:53:37,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:38,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:38,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:38,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:39,301 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:40,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:40,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:43,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:43,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:44,057 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:45,069 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:45,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:45,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:48,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:49,071 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:53:49,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:49,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:49,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:49,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:49,371 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:53:50,372 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:51,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:51,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:53,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:53,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:54,060 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:55,869 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:56,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:56,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:58,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:58,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:00,062 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:01,769 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:01,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:01,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:03,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:03,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:04,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:04,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:04,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:06,064 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:06,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:06,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:07,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:08,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:08,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:11,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:11,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:12,067 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:13,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:13,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:13,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:16,068 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:16,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:16,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:18,660 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:18,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:18,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:19,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:19,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:19,319 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:54:21,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:21,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:22,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:23,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:23,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:23,874 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:26,873 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:54:26,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:26,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:28,071 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:29,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:29,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:29,771 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:31,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:31,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:32,072 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:34,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:34,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:34,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:34,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:34,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:35,324 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:36,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:36,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:38,074 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:39,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:39,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:40,726 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:41,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:41,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:44,075 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:44,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:44,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:46,261 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:46,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:46,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:49,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:49,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:49,286 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:54:49,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:49,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:50,078 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:51,287 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:51,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:51,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:54,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:54,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,079 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:56,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,453 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:59,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:59,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:00,081 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:01,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:01,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:02,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:04,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:04,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:04,720 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:55:04,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:04,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:06,086 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:07,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:07,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:07,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:12,087 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:12,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:12,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:12,878 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:14,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:14,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:17,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:17,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:18,090 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:18,226 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:19,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:19,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:19,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:19,291 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:55:19,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:19,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:22,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:22,280 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:22,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,113 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:24,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:27,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:27,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:28,092 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:29,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:29,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:30,062 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:32,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:32,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:34,094 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:34,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:34,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:34,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:34,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:35,300 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:36,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:36,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:38,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:39,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:40,097 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:40,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:42,366 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:55:42,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:42,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:44,098 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:45,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:45,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:45,814 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:47,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:47,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:49,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:49,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:49,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:49,363 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:55:49,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:49,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:50,100 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:51,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:51,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:51,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:53,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:55:53,844 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:55:53,844 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:55:53,844 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:55:54,101 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:55:54,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:54,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:55,101 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:56,101 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:56,845 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:56,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:56,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:59,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:00,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:02,103 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:02,659 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:03,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:03,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:04,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:04,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:05,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:06,105 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:07,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:07,939 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:10,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:10,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:12,106 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:13,424 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:13,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:13,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:15,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:15,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:18,108 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:18,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:18,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:19,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:19,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:19,237 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:19,373 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:56:19,374 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:21,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:21,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:22,109 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:25,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:25,524 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:56:25,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:25,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:28,112 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:28,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:28,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:30,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:30,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:31,002 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:33,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:33,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:34,114 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:34,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:34,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:35,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:35,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:36,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:38,115 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:38,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:38,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:40,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:40,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:41,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:43,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:43,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:44,117 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:45,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:45,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:47,225 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:48,118 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:48,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:48,541 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:49,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:49,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:49,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:50,120 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:50,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:50,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:50,769 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:56:52,685 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:53,120 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:53,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:53,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:55,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:55,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:57,122 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:58,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:58,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:58,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:59,123 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:01,124 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:02,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:02,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:03,610 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:04,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:04,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:05,125 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:05,869 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:57:05,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:05,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:08,636 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:08,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:08,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:09,126 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:10,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:10,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:13,636 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:13,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:13,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:15,128 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:15,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:15,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:17,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:17,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:18,674 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:19,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:19,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:19,317 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:57:20,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:20,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:21,130 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:22,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:22,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:23,727 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:24,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:24,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:27,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:27,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:27,133 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:28,776 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:29,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:29,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:32,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:32,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:33,134 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:33,812 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:34,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:34,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:35,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:35,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:37,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:37,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:38,879 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:39,137 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:40,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:40,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:43,757 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:57:43,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:43,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:43,880 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:45,138 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:45,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:45,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:48,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:48,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:48,931 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:49,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:49,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:49,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:49,347 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:57:50,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:50,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:51,140 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:52,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:52,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:53,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:55,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:55,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:56,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:57:56,127 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:57:56,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:57:56,128 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:57:56,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:57:57,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:57,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:57,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:58,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:59,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:59,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:00,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:02,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:02,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:03,144 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:04,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:04,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:04,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:04,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:04,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:07,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:07,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:09,146 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:09,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:09,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:10,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:12,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:12,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:14,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:14,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:15,148 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:15,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:17,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:17,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:19,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:19,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:19,343 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:58:20,251 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:21,021 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:58:21,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:21,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:21,150 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:23,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:23,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:25,252 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:26,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:26,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:27,153 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:28,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:28,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:30,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:30,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:30,308 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:33,154 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:33,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:33,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:34,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:34,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:35,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:35,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:35,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:37,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:37,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:40,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:40,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:40,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:41,157 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:42,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:42,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:45,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:45,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:45,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:47,159 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:47,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:47,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:49,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:49,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:49,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:49,299 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:58:50,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:50,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:50,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:52,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:52,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:53,161 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:54,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:54,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:55,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:58,413 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:58:58,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:58,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:59,163 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:00,612 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:01,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:01,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:03,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:03,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:04,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:59:04,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:59:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:59:05,165 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:06,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:06,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:06,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:08,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:08,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:11,167 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:11,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:11,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:11,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:13,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:13,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:16,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:16,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:16,810 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:17,169 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:18,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:18,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:19,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:59:19,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:59:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:59:19,345 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:59:21,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:21,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:21,863 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:23,171 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:23,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:23,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:26,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:26,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:26,864 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:28,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:28,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:29,173 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:31,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:31,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:31,911 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:33,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:33,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:34,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:59:34,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:59:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:59:35,175 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:37,253 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:59:37,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:37,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:37,382 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:40,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:40,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:41,177 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:42,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:42,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:43,045 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:44,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:44,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:46,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:46,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:47,179 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:48,103 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:49,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:59:49,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:59:49,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:59:49,293 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:59:49,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:49,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:52,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:52,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:53,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:53,181 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:55,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:55,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:57,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:57,076 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:58,210 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:59,183 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:59,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:00,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:02,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:02,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:03,211 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:04,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:00:04,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:00:04,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:00:05,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:05,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:05,185 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:07,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:07,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:08,703 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:09,187 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:10,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:10,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:11,187 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:12,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:12,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:14,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:15,189 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:15,694 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:00:15,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:00:15,945 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:00:15,945 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:00:15,946 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:00:16,189 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:00:16,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:16,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:17,190 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:18,190 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:18,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:18,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:19,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:00:19,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:00:19,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:00:21,090 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:00:21,091 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:21,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:21,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:23,191 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:23,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:23,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:26,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:26,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:26,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:27,193 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:29,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:29,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:31,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:31,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:31,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:33,194 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:34,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:00:34,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:00:34,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:00:34,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:34,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:36,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:36,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:36,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:38,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:38,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:39,197 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:41,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:41,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:42,251 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:43,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:43,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:45,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:46,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:46,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:47,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:48,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:48,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:49,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:00:49,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:00:49,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:00:50,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:50,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:50,935 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:00:52,936 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:53,202 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:53,999 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:00:54,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:54,040 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:57,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:57,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:57,203 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:58,445 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:59,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:59,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:01,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:01,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:03,205 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:03,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:04,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:01:04,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:01:04,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:01:04,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:04,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:06,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:06,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:08,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:09,207 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:09,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:09,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:11,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:11,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:13,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:14,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:14,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:15,209 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:16,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:16,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:18,659 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:19,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:01:19,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:01:19,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:01:19,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:19,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:20,440 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:01:21,211 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:21,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:21,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:24,442 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:24,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:24,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:26,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:26,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:27,213 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:28,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:28,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:29,797 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:32,116 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:01:32,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:32,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:33,215 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:34,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:01:34,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:01:34,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:01:34,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:34,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:35,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:36,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:36,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:39,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:39,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:39,217 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:40,933 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:41,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:41,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:43,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:43,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:45,219 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:45,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:45,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:46,001 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:48,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:48,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:49,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:01:49,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:01:49,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:01:49,288 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:01:50,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:50,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:51,069 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:51,221 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:53,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:53,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:55,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:55,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:56,070 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:57,222 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:57,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:58,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:01,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:01,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:01,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:03,224 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:03,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:03,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:04,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:02:04,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:02:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:02:05,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:05,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:06,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:08,739 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:02:08,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:08,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:09,226 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:11,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:11,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:12,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:13,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:13,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:15,229 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:15,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:15,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:17,341 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:18,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:18,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:19,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:02:19,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:02:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:02:19,264 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:02:20,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:20,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:21,230 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:22,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:22,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:22,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:24,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:24,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:26,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:26,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:27,232 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:27,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:28,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:28,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:31,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:31,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:32,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:33,235 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:33,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:33,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:34,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:02:34,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:02:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:02:36,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:36,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:38,330 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:38,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:38,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:39,237 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:41,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:41,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:43,633 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:44,785 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:02:44,786 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:02:44,786 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:02:44,788 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:02:45,240 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:02:45,633 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:02:45,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:45,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:46,241 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:47,241 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:47,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:47,732 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:48,789 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:49,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:02:49,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:02:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:02:49,755 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:49,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:50,723 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:02:53,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:53,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:53,243 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:53,792 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:55,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:55,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:58,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:58,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:58,896 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:59,245 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:00,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:00,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:03,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:03,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:03,961 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:04,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:03:04,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:03:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:03:05,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:05,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:05,247 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:08,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:08,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:09,036 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:10,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:10,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:11,249 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:12,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:12,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:14,036 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:14,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:14,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:17,251 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:17,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:17,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:19,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:19,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:03:19,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:03:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:03:19,316 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:03:19,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:19,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:23,161 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:03:23,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:23,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:23,253 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:24,214 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:26,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:26,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:28,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:28,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:29,255 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:29,301 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:31,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:31,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:33,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:33,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:34,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:03:34,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:03:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:03:34,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:35,258 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:35,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:35,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:37,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:37,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:39,355 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:39,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:39,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:41,260 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:42,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:42,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:44,467 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:44,679 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:44,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:47,262 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:47,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:47,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:49,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:03:49,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:03:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:03:49,331 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:03:49,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:49,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:50,332 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:52,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:52,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:53,264 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:54,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:54,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:55,637 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:56,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:56,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:59,267 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:00,560 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:04:00,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:00,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:00,719 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:03,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:03,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:04,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:04:04,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:04:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:04:05,269 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:05,751 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:05,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:06,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:07,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:07,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:10,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:10,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:11,272 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:11,909 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:12,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:12,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:15,800 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:15,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:17,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:17,274 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:17,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:17,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:19,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:04:19,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:04:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:04:19,369 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:04:20,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:20,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:22,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:22,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:22,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:23,276 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:24,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:25,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:27,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:28,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:28,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:29,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:30,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:30,280 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:32,220 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:32,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:32,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:34,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:04:34,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:04:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:04:35,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:35,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:35,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:37,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:39,094 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:04:39,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:39,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:41,283 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:41,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:41,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:42,435 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:44,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:44,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:46,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:46,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:47,285 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:47,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:48,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:48,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:49,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:04:49,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:04:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:04:50,417 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:04:51,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:51,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:52,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:53,287 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:53,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:53,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:55,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:55,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:57,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:58,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:58,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:59,289 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:00,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:00,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:02,749 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:03,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:03,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:04,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:05:04,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:05:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:05:05,292 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:05,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:05,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:07,840 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:08,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:08,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:11,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:11,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:12,948 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:13,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:13,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:13,295 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:16,902 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:05:16,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:16,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:17,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:05:17,120 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:05:17,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:05:17,122 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:05:17,296 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:05:18,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:18,297 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:18,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:18,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:19,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:05:19,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:05:19,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:05:19,297 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:19,344 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:05:21,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:22,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:23,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:24,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:24,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:25,299 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:27,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:27,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:28,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:29,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:29,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:31,301 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:32,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:32,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:33,265 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:34,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:05:34,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:05:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:05:34,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:34,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:36,235 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:36,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:37,303 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:38,355 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:39,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:39,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:41,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:41,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:43,305 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:43,446 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:43,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:43,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:46,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:46,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:48,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:48,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:48,556 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:49,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:05:49,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:05:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:05:49,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:05:49,307 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:51,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:51,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:53,641 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:55,207 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:05:55,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:55,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:55,309 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:57,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:57,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:58,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:59,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:59,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:01,311 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:01,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:01,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:03,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:03,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:03,728 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:04,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:06:04,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:06:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:06:05,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:05,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:07,313 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:07,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:07,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:08,823 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:10,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:10,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:12,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:12,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:13,315 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:13,907 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:15,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:15,541 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:17,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:17,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:18,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:19,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:06:19,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:06:19,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:06:19,317 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:20,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:20,674 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:06:20,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:22,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:22,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:24,080 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:24,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:24,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:25,319 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:27,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:28,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:29,082 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:31,320 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:31,842 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:06:31,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:31,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:33,321 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:33,902 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:33,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:34,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:06:34,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:06:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:06:34,305 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:36,767 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:36,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:37,323 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:38,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:38,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:39,324 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:40,109 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:41,324 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:41,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:41,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:43,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:43,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:45,603 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:45,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:45,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:48,637 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:48,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:49,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:06:49,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:06:49,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:06:49,282 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:06:50,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:50,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:51,283 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:53,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:53,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:56,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:56,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:56,284 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:59,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:59,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:59,331 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:01,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:01,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:01,472 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:04,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:07:04,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:07:04,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:07:04,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:04,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:05,333 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:06,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:06,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:07,350 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:10,114 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:07:10,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:10,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:11,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:12,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:13,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:13,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:15,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:15,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:17,337 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:17,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:17,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:17,753 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:19,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:07:19,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:07:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:07:19,374 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:07:19,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:19,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:22,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:22,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:22,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:23,340 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:24,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:24,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:27,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:27,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:27,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:29,342 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:29,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:29,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:32,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:32,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:32,926 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:34,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:07:34,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:07:34,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:07:34,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:34,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:35,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:36,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:36,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:38,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:38,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:39,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:41,346 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:41,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:41,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:43,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:43,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:44,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:46,281 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:07:46,283 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:07:46,283 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:07:46,283 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:07:46,348 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:07:47,348 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:47,468 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:07:47,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:47,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:48,349 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:49,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:07:49,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:07:49,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:07:49,345 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:07:49,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:50,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:50,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:52,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:52,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:53,350 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:55,305 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:55,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:55,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:57,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:57,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:59,352 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:59,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:59,732 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:00,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:02,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:02,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:04,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:08:04,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:08:04,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:08:04,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:04,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:05,354 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:05,488 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:07,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:07,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:09,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:09,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:10,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:11,357 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:12,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:12,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:14,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:15,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:15,784 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:17,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:17,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:17,359 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:19,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:08:19,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:08:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:08:19,345 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:08:20,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:20,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:21,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:22,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:22,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:23,361 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:26,119 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:08:26,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:26,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:26,955 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:28,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:28,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:30,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:30,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:31,364 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:32,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:33,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:33,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:34,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:08:34,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:08:34,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:08:35,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:35,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:37,366 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:37,497 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:38,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:38,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:40,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:40,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:42,580 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:43,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:43,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:43,368 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:45,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:45,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:47,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:47,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:47,580 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:49,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:08:49,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:08:49,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:08:49,370 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:49,373 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:08:50,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:50,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:52,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:52,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:53,374 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:54,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:54,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:55,372 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:57,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:57,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:58,774 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:59,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:59,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:01,374 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:03,154 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:09:03,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:03,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:03,866 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:04,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:09:04,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:09:04,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:09:06,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:06,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:07,377 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:08,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:08,105 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:08,946 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:10,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:10,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:13,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:13,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:13,379 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:14,052 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:15,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:15,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:17,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:17,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:19,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:19,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:09:19,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:09:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:09:19,370 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:09:19,381 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:19,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:19,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:22,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:22,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:24,244 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:24,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:24,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:25,384 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:27,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:27,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:29,245 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:29,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:29,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:31,386 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:32,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:32,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:34,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:09:34,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:09:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:09:34,316 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:34,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:34,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:36,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:36,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:37,388 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:39,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:40,769 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:09:40,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:40,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:43,390 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:43,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:43,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:44,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:45,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:45,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:48,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:49,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:49,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:09:49,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:09:49,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:09:49,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:09:49,393 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:50,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:51,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:51,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:53,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:53,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:55,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:55,720 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:56,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:56,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:58,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:58,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:00,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:00,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:00,721 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:01,397 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:03,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:03,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:04,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:10:04,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:10:04,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:10:05,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:05,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:06,310 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:07,399 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:07,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:07,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:10,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:10,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:11,928 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:12,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:12,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:13,402 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:15,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:15,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:17,018 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:19,141 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:10:19,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:10:19,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:10:19,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:19,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:10:19,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:19,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:10:19,315 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:10:19,315 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:10:19,316 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:10:19,316 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:10:19,404 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:10:19,404 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:20,404 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:21,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:21,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:21,404 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:22,318 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:23,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:23,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:25,406 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:26,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:26,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:28,207 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:28,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:28,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:30,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:30,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:33,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:33,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:33,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:34,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:10:34,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:10:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:10:34,409 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:35,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:35,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:38,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:38,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:38,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:40,411 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:40,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:40,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:43,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:43,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:43,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:45,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:45,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:46,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:47,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:47,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:48,488 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:49,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:10:49,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:10:49,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:10:50,581 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:10:51,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:51,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:52,415 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:53,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:53,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:53,583 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:56,825 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:10:56,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:56,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:58,418 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:58,700 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:59,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:59,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:01,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:01,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:03,794 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:04,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:11:04,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:11:04,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:11:04,419 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:04,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:04,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:06,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:06,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:08,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:08,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:08,896 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:10,421 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:11,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:11,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:13,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:13,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:13,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:16,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:16,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:16,424 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:18,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:19,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:11:19,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:11:19,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:11:19,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:19,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:19,320 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:11:21,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:21,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:22,426 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:24,096 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:24,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:24,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:26,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:26,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:28,428 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:29,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:29,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:29,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:31,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:31,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:11:34,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:11:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:11:34,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:34,430 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:35,001 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:11:35,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:35,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:38,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:38,172 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:39,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:40,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:40,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:40,432 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:42,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:42,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:44,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:45,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:45,203 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:46,434 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:47,227 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:47,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:49,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:11:49,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:11:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:11:49,359 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:11:50,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:50,130 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:50,360 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:52,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:52,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:52,436 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:55,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:55,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:55,600 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:57,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:57,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:58,438 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:59,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:59,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:00,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:02,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:02,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:04,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:12:04,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:12:04,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:12:04,441 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:04,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:04,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:06,335 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:06,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:06,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:09,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:09,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:10,443 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:11,905 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:13,031 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:12:13,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:13,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:16,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:16,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:16,445 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:17,010 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:18,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:18,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:19,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:12:19,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:12:19,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:12:19,351 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:12:20,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:20,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:22,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:22,447 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:23,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:23,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:25,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:25,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:27,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:27,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:27,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:30,450 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:30,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:30,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:32,384 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:32,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:32,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:12:34,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:12:34,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:12:35,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:35,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:36,452 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:37,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:37,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:38,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:40,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:40,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:42,454 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:42,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:42,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:43,571 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:44,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:46,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:48,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:48,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:48,456 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:48,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:49,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:12:49,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:12:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:12:49,382 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:12:51,634 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:12:51,636 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:12:51,636 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:12:51,637 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:12:51,854 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:12:51,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:51,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:52,457 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:12:52,457 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:53,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:53,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:54,458 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:54,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:56,459 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:56,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:56,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:58,459 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:58,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:58,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:00,035 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:01,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:01,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:02,461 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:03,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:03,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:04,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:13:04,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:13:04,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:13:04,461 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:05,313 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:06,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:06,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:08,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:08,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:10,463 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:11,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:11,036 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:11,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:13,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:13,073 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:15,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:15,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:16,442 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:16,465 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:18,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:18,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:19,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:13:19,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:13:19,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:13:19,354 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:13:20,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:20,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:21,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:22,467 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:23,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:23,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:25,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:25,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:26,649 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:28,469 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:28,823 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:13:28,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:28,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:31,649 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:32,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:32,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:34,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:34,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:13:34,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:13:34,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:13:34,470 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:36,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:36,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:37,370 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:38,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:38,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:40,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:40,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:40,472 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:42,830 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:43,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:43,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:45,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:45,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:46,474 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:47,933 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:48,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:48,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:13:49,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:13:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:13:50,351 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:13:50,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:50,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:52,476 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:53,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:53,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:53,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:55,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:55,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:58,479 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:58,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:58,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:59,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:00,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:00,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:02,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:02,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:04,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:14:04,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:14:04,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:14:04,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:04,481 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:06,618 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:14:06,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:06,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:09,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:09,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:09,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:10,483 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:11,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:11,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:14,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:14,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:14,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:16,485 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:16,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:16,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:19,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:14:19,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:14:19,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:14:19,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:19,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:21,187 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:14:21,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:21,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:21,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:22,487 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:23,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:23,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:26,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:27,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:27,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:28,489 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:29,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:29,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:31,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:31,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:31,631 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:34,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:14:34,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:14:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:14:34,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:34,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:34,491 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:36,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:36,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:37,283 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:39,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:39,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:40,492 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:41,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:41,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:42,839 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:45,034 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:14:45,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:45,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:46,494 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:47,932 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:48,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:48,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:14:49,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:14:49,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:14:49,371 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:14:50,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:50,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:52,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:52,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:52,496 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:53,048 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:55,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:55,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:57,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:57,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:58,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:58,498 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:00,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:00,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:02,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:02,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:03,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:04,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:15:04,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:15:04,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:15:04,500 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:05,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:05,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:07,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:07,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:08,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:09,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:15:09,414 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:15:09,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:15:09,415 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:15:09,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:15:09,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:09,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:10,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:11,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:11,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:13,416 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:14,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:15,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:16,504 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:17,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:17,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:18,450 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:19,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:19,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:19,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:15:19,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:15:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:15:19,364 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:15:22,506 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:22,574 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:15:22,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:22,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:23,572 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:25,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:25,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:27,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:27,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:28,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:30,509 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:30,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:30,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:32,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:32,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:33,669 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:34,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:15:34,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:15:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:15:34,637 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:34,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:36,511 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:36,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:36,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:39,308 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:39,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:39,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:41,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:41,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:42,513 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:44,881 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:45,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:45,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:47,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:47,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:48,515 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:15:49,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:15:49,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:15:49,310 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:15:50,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:50,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:50,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:52,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:52,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:54,517 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:55,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:55,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:56,092 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:57,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:57,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:00,519 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:01,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:01,266 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:16:01,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:01,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:03,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:03,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:04,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:16:04,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:16:04,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:16:06,288 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:06,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:06,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:06,522 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:08,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:08,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:11,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:11,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:11,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:12,524 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:13,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:13,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:16,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:16,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:16,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:18,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:18,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:18,526 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:19,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:16:19,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:16:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:16:20,303 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:16:20,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:20,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:22,305 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:23,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:23,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:24,529 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:25,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:25,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:27,602 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:27,755 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:27,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:29,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:29,837 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:30,531 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:32,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:32,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:32,724 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:34,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:16:34,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:16:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:16:34,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:34,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:36,533 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:37,830 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:38,258 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:16:38,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:38,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:41,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:41,090 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:42,535 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:42,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:43,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:43,203 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:45,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:45,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:47,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:47,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:48,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:48,537 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:49,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:16:49,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:16:49,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:16:49,325 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:16:49,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:49,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:52,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:52,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:53,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:54,540 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:54,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:54,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:56,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:56,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:58,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:59,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:59,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:00,542 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:01,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:01,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:03,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:04,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:17:04,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:17:04,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:17:04,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:04,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:06,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:06,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:06,544 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:08,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:08,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:08,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:10,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:10,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:12,546 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:13,674 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:14,374 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:17:14,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:14,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:17,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:17,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:18,789 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:19,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:17:19,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:17:19,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:17:19,323 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:17:19,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:19,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:20,549 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:22,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:22,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:23,905 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:24,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:24,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:26,551 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:27,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:27,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:28,906 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:29,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:29,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:31,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:31,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:32,554 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:33,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:33,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:34,011 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:34,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:17:34,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:17:34,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:17:37,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:37,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:38,556 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:39,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:39,130 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:39,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:41,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:41,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:42,313 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:17:42,314 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:17:42,315 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:17:42,316 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:17:42,557 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:17:43,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:43,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:44,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:44,558 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:45,558 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:46,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:46,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:48,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:48,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:17:49,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:17:49,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:17:50,508 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:17:50,509 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:50,560 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:51,856 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:17:51,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:51,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:54,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:54,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:56,476 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:56,562 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:56,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:56,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:58,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:58,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:01,576 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:02,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:02,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:02,564 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:04,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:18:04,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:18:04,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:18:04,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:04,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:06,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:06,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:06,695 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:08,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:09,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:09,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:11,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:11,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:11,696 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:14,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:14,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:14,568 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:16,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:16,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:16,813 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:19,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:18:19,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:18:19,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:18:19,372 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:18:19,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:19,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:20,570 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:21,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:21,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:22,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:23,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:23,657 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:25,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:25,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:26,572 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:28,031 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:29,624 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:18:29,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:29,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:32,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:32,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:32,574 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:33,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:34,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:18:34,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:18:34,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:18:34,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:34,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:37,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:37,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:38,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:38,576 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:39,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:39,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:42,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:42,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:43,369 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:44,578 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:44,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:44,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:46,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:46,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:48,466 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:18:49,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:18:49,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:18:50,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:50,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:50,613 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:18:52,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:52,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:52,581 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:53,615 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:54,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:54,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:56,582 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:57,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:57,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:58,686 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:59,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:59,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:02,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:02,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:02,584 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:03,804 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:04,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:19:04,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:19:04,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:19:04,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:04,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:08,227 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:19:08,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:08,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:08,904 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:10,586 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:11,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:11,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:13,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:13,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:13,905 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:15,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:15,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:16,589 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:17,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:17,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:19,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:19:19,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:19:19,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:19:19,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:19,316 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:19:20,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:20,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:20,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:22,591 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:22,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:22,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:24,591 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:24,826 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:25,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:25,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:27,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:27,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:28,593 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:30,326 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:30,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:30,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:32,594 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:32,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:32,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:34,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:19:34,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:19:34,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:19:34,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:34,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:35,816 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:36,596 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:36,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:36,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:40,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:40,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:40,816 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:42,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:42,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:44,599 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:45,806 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:19:45,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:45,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:45,930 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:48,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:48,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:19:49,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:19:49,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:19:49,375 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:19:50,601 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:50,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:50,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:51,377 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:53,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:53,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:55,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:55,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:56,603 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:57,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:57,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:57,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:59,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:00,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:20:00,326 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:20:00,327 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:20:00,327 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:20:00,604 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:20:00,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:01,605 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:02,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:02,605 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:02,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:02,902 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:20:04,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:20:04,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:20:04,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:04,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:06,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:06,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:07,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:08,607 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:10,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:10,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:12,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:12,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:12,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:14,610 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:15,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:15,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:17,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:17,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:17,483 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:20:19,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:20:19,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:20:20,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:20,368 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:20:20,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:20,612 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:23,370 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:24,239 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:20:24,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:24,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:26,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:26,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:26,613 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:28,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:28,672 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:29,595 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:31,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:31,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:32,615 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:33,792 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:20:34,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:20:34,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:20:34,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:34,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:36,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:36,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:38,617 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:38,896 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:39,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:39,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:41,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:41,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:43,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:43,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:43,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:44,619 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:45,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:45,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:48,999 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:49,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:49,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:20:49,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:20:49,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:20:50,341 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:20:50,620 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:51,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:51,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:53,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:53,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:54,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:56,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:56,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:56,622 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:58,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:58,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:00,213 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:01,754 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:21:01,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:01,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:02,625 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:21:04,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:21:04,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:21:04,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:04,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:05,355 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:06,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:06,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:08,627 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:08,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:08,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:10,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:10,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:10,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:12,866 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:12,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:14,629 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:15,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:16,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:16,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:18,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:18,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:21:19,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:21:19,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:21:19,344 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:21:20,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:20,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:20,631 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:20,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:23,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:23,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:25,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:25,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:25,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:26,633 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:28,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:28,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:30,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:30,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:30,780 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:32,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:32,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:32,635 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:21:34,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:21:34,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:21:35,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:35,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:36,383 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:38,636 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:39,081 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:21:39,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:39,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:41,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:41,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:42,006 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:43,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:43,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:45,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:45,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:46,639 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:47,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:47,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:47,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:21:49,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:21:49,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:21:49,384 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:21:51,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:51,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:52,242 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:52,641 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:53,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:53,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:56,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:56,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:57,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:58,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:58,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:58,643 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:00,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:00,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:02,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:02,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:02,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:22:04,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:22:04,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:22:04,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:04,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:04,645 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:07,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:07,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:07,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:09,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:09,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:10,647 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:12,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:12,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:12,573 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:16,146 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:22:16,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:16,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:16,649 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:17,688 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:18,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:18,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:22:19,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:22:19,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:22:19,367 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:22:21,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:21,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:22,651 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:22,791 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:23,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:23,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:25,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:25,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:27,902 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:28,653 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:28,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:28,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:30,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:30,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:32,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:32,904 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:33,086 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:22:33,088 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:22:33,088 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:22:33,088 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:22:33,654 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:22:33,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:22:34,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:22:34,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:22:34,655 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:35,655 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:35,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:35,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:38,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:38,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:38,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:40,657 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:40,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:40,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:44,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:44,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:44,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:46,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:46,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:46,659 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:48,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:48,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:49,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:22:49,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:22:49,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:22:49,280 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:49,328 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:22:50,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:50,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:52,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:53,909 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:22:53,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:53,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:54,776 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:56,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:56,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:58,663 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:58,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:58,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:59,879 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:01,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:01,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:03,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:03,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:04,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:23:04,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:23:04,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:23:04,665 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:05,296 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:06,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:07,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:09,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:09,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:10,666 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:11,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:11,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:11,116 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:13,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:13,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:16,117 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:16,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:16,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:18,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:18,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:18,669 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:23:19,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:23:19,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:23:19,329 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:23:21,331 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:21,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:21,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:23,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:23,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:24,671 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:26,350 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:26,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:26,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:28,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:28,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:30,673 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:31,457 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:32,263 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:23:32,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:32,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:23:34,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:23:34,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:23:35,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:35,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:36,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:36,675 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:37,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:37,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:39,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:39,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:41,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:41,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:41,690 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:42,677 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:44,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:44,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:46,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:46,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:46,691 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:48,680 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:49,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:23:49,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:23:49,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:23:49,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:49,332 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:23:49,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:51,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:51,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:52,333 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:54,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:54,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:54,682 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:56,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:56,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:57,900 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:58,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:58,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:00,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:00,684 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:00,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:03,032 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:03,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:03,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:04,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:24:04,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:24:04,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:24:05,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:05,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:06,686 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:08,496 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:09,670 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:24:09,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:09,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:12,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:12,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:12,688 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:13,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:14,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:14,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:17,414 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:17,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:18,690 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:18,715 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:24:19,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:24:19,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:24:19,341 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:24:19,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:19,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:21,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:21,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:23,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:23,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:23,829 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:24,693 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:26,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:26,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:28,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:28,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:28,829 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:30,694 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:30,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:30,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:33,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:33,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:33,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:24:34,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:24:34,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:24:35,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:35,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:36,696 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:38,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:38,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:39,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:40,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:40,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:42,698 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:44,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:44,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:44,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:47,815 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:24:47,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:47,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:24:49,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:24:49,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:24:49,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:24:49,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:49,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:49,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:50,701 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:53,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:53,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:54,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:55,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:55,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:56,703 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:58,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:58,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:59,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:00,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:00,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:02,705 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:03,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:03,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:04,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:25:04,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:25:04,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:25:05,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:05,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:05,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:06,685 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:25:06,687 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:25:06,687 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:25:06,687 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:25:06,706 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:25:07,707 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:08,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:08,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:08,707 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:10,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:10,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:10,688 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:12,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:12,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:14,709 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:15,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:15,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:15,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:17,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:17,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:19,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:25:19,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:25:19,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:25:19,397 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:25:19,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:19,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:20,711 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:21,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:22,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:22,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:26,233 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:25:26,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:26,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:26,712 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:26,971 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:29,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:29,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:31,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:31,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:32,089 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:32,715 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:33,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:33,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:34,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:25:34,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:25:34,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:25:35,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:35,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:37,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:37,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:37,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:38,717 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:39,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:40,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:42,207 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:42,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:42,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:44,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:44,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:44,719 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:46,720 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:47,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:47,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:47,813 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:48,720 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:25:49,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:25:49,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:25:49,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:49,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:50,811 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:25:52,722 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:52,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:52,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:53,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:54,722 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:54,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:55,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:57,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:57,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:58,723 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:58,904 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:59,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:59,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:02,725 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:02,870 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:26:02,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:02,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:04,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:26:04,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:26:04,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:26:04,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:04,725 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:04,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:04,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:06,726 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:08,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:08,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:10,100 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:10,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:10,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:10,728 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:13,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:13,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:15,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:15,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:15,101 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:16,730 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:17,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:17,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:19,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:26:19,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:26:19,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:26:19,340 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:26:19,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:19,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:20,341 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:22,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:22,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:22,731 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:25,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:25,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:25,872 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:27,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:27,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:28,733 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:29,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:29,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:31,847 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:32,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:32,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:34,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:26:34,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:26:34,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:26:34,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:34,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:34,735 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:37,349 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:37,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:37,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:38,737 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:40,971 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:26:40,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:41,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:42,648 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:43,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:43,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:44,738 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:46,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:46,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:48,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:48,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:48,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:49,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:26:49,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:26:49,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:26:49,359 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:26:50,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:50,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:50,741 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:53,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:53,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:54,251 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:54,742 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:55,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:55,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:57,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:57,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:59,252 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:00,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:00,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:00,744 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:02,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:02,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:04,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:27:04,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:27:04,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:27:04,327 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:05,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:27:05,185 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:27:05,186 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:27:05,186 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:27:05,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:05,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:05,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:27:05,746 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:06,746 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:07,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:07,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:09,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:09,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:10,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:12,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:12,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:12,749 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:14,679 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:14,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:16,053 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:17,751 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:18,389 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:27:18,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:18,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:19,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:27:19,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:27:19,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:27:19,344 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:27:21,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:21,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:21,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:23,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:23,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:23,753 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:25,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:25,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:26,611 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:28,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:28,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:29,755 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:30,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:30,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:32,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:32,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:32,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:34,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:27:34,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:27:34,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:27:35,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:35,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:35,756 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:37,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:37,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:38,281 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:39,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:39,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:39,758 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:42,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:42,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:43,282 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:44,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:44,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:45,760 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:47,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:47,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:49,072 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:49,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:27:49,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:27:49,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:27:49,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:49,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:49,379 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:27:51,762 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:52,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:52,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:54,381 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:55,988 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:27:56,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:56,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:57,764 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:58,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:58,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:59,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:01,242 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:01,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:01,765 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:03,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:03,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:04,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:28:04,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:28:04,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:28:05,291 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:05,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:05,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:07,767 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:08,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:08,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:10,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:10,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:10,848 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:13,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:13,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:13,769 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:15,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:15,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:16,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:17,770 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:18,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:18,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:19,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:28:19,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:28:19,180 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:28:19,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:28:20,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:20,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:21,378 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:22,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:22,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:25,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:25,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:26,379 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:27,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:27,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:29,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:29,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:31,380 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:32,971 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:28:32,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:33,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:34,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:28:34,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:28:34,180 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:28:35,776 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:35,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:35,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:37,363 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:37,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:37,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:40,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:40,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:41,778 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:42,643 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:42,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:42,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:44,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:44,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:45,779 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:48,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:48,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:48,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:49,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:28:49,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:28:49,180 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:28:50,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:50,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:50,524 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:28:51,781 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:52,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:52,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:53,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:55,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:55,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:57,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:57,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:57,782 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:58,979 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:00,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:00,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:02,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:02,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:03,784 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:04,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:29:04,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:29:04,180 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:29:04,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:04,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:04,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:07,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:07,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:07,785 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:09,372 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:10,753 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:29:10,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:10,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:12,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:12,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:13,787 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:14,733 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:16,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:16,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:18,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:18,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:19,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:29:19,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:29:19,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:29:19,789 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:20,447 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:29:20,448 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:21,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:21,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:23,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:23,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:23,791 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:26,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:26,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:26,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:28,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:28,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:29,794 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:30,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:30,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:32,258 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:33,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:33,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:34,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:29:34,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:29:34,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:29:34,218 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:29:34,320 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:29:34,320 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:29:34,320 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:29:34,795 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:29:35,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:35,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:35,796 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:37,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:37,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:37,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:40,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:40,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:41,798 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:42,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:42,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:43,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:45,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:45,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:47,800 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:49,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:49,125 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:29:49,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:49,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:49,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:29:49,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:29:49,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:29:49,358 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:29:51,801 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:52,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:52,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:54,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:54,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:54,984 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:56,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:56,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:57,803 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:59,242 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:59,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:00,944 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:01,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:01,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:03,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:03,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:03,805 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:04,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:30:04,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:30:04,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:30:06,288 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:06,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:06,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:08,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:08,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:09,807 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:11,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:11,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:11,742 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:13,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:13,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:15,809 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:16,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:16,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:17,678 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:18,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:18,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:19,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:30:19,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:30:19,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:30:19,353 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:30:19,810 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:20,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:20,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:23,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:23,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:23,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:25,813 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:27,160 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:30:27,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:27,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:28,438 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:29,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:29,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:31,815 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:32,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:32,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:34,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:34,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:34,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:30:34,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:30:34,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:30:35,816 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:37,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:37,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:39,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:39,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:39,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:41,818 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:42,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:42,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:44,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:44,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:45,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:46,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:46,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:47,820 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:48,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:48,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:49,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:30:49,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:30:49,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:30:50,817 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:30:50,818 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:51,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:51,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:53,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:53,822 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:53,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:55,916 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:56,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:56,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:58,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:58,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:59,824 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:01,711 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:01,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:01,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:03,825 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:04,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:31:04,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:31:04,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:31:05,554 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:31:05,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:05,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:07,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:07,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:07,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:09,827 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:10,637 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:10,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:12,643 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:12,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:12,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:14,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:14,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:15,829 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:16,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:16,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:18,261 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:19,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:31:19,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:31:19,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:31:19,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:19,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:19,831 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:20,385 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:31:21,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:21,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:23,386 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:24,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:24,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:25,833 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:26,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:26,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:28,460 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:29,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:29,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:31,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:31,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:31,834 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:33,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:33,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:34,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:31:34,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:31:34,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:31:34,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:35,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:35,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:37,836 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:39,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:39,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:39,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:42,686 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:31:42,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:42,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:43,838 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:44,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:44,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:45,074 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:47,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:47,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:47,840 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:49,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:31:49,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:31:49,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:31:49,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:31:49,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:49,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:50,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:52,215 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:31:52,216 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:31:52,216 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:31:52,217 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:31:52,842 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:31:52,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:52,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:53,842 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:54,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:54,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:56,218 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:56,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:56,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:59,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:59,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:59,844 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:01,868 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:02,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:02,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:04,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:32:04,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:32:04,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:32:04,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:04,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:05,846 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:06,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:07,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:07,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:09,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:09,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:11,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:11,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:11,849 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:12,778 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:14,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:14,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:15,850 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:16,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:16,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:18,600 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:19,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:32:19,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:32:19,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:32:19,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:32:20,124 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:32:20,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:20,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:21,852 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:22,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:22,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:24,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:25,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:25,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:25,853 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:27,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:27,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:29,650 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:30,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:30,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:31,856 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:32,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:32,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:34,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:32:34,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:32:34,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:32:35,386 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:35,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:35,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:37,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:37,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:37,858 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:40,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:40,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:40,993 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:42,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:42,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:43,860 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:44,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:44,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:45,993 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:47,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:48,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:49,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:32:49,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:32:49,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:32:49,361 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:32:49,862 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:50,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:50,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:51,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:52,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:52,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:53,863 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:55,100 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:55,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:56,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:58,793 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:32:58,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:58,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:59,865 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:01,734 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:01,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:01,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:03,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:03,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:33:04,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:33:04,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:33:05,866 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:06,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:06,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:07,381 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:08,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:08,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:09,869 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:10,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:10,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:12,483 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:13,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:13,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:15,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:15,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:15,870 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:18,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:18,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:18,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:19,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:33:19,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:33:19,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:33:19,387 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:33:20,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:20,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:21,872 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:23,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:23,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:24,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:25,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:25,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:27,874 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:28,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:28,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:29,282 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:30,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:30,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:33,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:33,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:33,877 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:34,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:33:34,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:33:34,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:33:34,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:36,644 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:33:36,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:36,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:37,878 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:38,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:38,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:40,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:41,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:41,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:43,414 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:43,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:43,881 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:45,978 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:46,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:46,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:48,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:48,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:49,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:33:49,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:33:49,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:33:49,364 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:33:49,883 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:51,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:51,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:51,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:53,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:53,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:53,885 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:56,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:56,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:56,726 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:58,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:58,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:59,886 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:01,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:01,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:02,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:03,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:03,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:34:04,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:34:04,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:34:05,889 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:05,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:06,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:08,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:08,040 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:08,335 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:09,386 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:34:09,388 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:34:09,388 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:34:09,389 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:34:09,890 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:34:09,891 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:10,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:10,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:11,891 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:13,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:14,714 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:34:14,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:14,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:17,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:17,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:17,893 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:34:19,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:34:19,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:34:19,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:19,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:20,963 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:34:20,964 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:22,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:22,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:23,895 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:24,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:24,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:25,965 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:27,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:27,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:27,896 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:29,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:29,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:31,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:32,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:32,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:33,898 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:34,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:34:34,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:34:34,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:34:34,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:34,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:36,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:37,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:37,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:37,899 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:39,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:39,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:41,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:41,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:41,810 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:43,901 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:44,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:44,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:46,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:46,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:47,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:49,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:34:49,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:34:49,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:34:49,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:49,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:49,903 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:50,835 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:34:52,836 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:52,927 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:34:52,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:52,961 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:55,905 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:55,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:56,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:58,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:58,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:58,453 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:59,906 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:00,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:00,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:02,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:02,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:03,734 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:35:04,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:35:04,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:35:05,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:05,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:05,909 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:07,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:07,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:09,370 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:10,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:10,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:11,912 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:12,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:12,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:14,984 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:15,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:15,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:15,914 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:17,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:17,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:35:19,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:35:19,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:35:20,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:20,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:20,942 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:35:20,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:21,916 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:22,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:22,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:25,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:25,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:25,945 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:27,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:27,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:27,917 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:30,750 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:35:30,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:30,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:31,318 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:33,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:33,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:33,919 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:34,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:35:34,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:35:34,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:35:35,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:35,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:36,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:38,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:38,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:39,921 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:40,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:40,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:41,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:42,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:42,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:43,922 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:45,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:45,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:47,289 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:47,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:47,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:49,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:35:49,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:35:49,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:35:49,925 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:50,432 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:35:50,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:50,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:52,433 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:52,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:52,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:55,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:55,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:55,927 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:57,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:57,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:57,845 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:59,929 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:00,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:00,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:02,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:02,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:03,681 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:36:04,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:36:04,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:36:05,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:05,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:05,931 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:08,841 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:36:08,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:08,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:09,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:11,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:11,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:11,933 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:13,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:13,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:14,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:15,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:15,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:17,936 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:18,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:18,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:36:19,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:36:19,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:36:19,372 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:36:20,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:20,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:20,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:22,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:22,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:23,943 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:25,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:25,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:25,910 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:25,981 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:36:25,983 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:36:25,983 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:36:25,983 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:36:26,947 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:36:27,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:27,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:27,947 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:30,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:30,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:30,984 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:32,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:32,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:33,949 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:34,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:36:34,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:36:34,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:36:35,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:35,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:36,380 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:37,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:37,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:37,950 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:40,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:40,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:41,996 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:42,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:42,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:43,953 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:46,224 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:36:46,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:46,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:47,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:49,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:49,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:49,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:36:49,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:36:49,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:36:49,955 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:50,292 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:36:51,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:51,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:53,000 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:53,957 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:53,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:53,989 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:56,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:56,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:58,001 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:58,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:58,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:59,959 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:00,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:00,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:02,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:02,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:03,849 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:04,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:37:04,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:37:04,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:37:04,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:04,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:05,961 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:07,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:07,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:09,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:09,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:09,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:11,963 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:12,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:12,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:14,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:14,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:14,626 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:17,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:17,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:17,964 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:19,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:37:19,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:37:19,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:37:19,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:19,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:19,375 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:37:20,201 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:21,966 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:22,708 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:37:22,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:22,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:25,202 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:25,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:25,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:27,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:27,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:27,967 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:30,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:30,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:31,046 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:32,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:32,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:33,969 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:34,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:37:34,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:37:34,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:37:35,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:35,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:36,291 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:37,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:37,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:37,971 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:40,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:40,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:41,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:42,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:42,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:43,972 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:45,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:45,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:47,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:47,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:47,611 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:49,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:37:49,165 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:37:49,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:37:49,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:37:49,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:49,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:49,974 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:51,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:52,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:53,378 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:55,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:55,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:55,976 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:57,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:57,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:58,449 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:00,395 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:38:00,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:00,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:01,978 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:03,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:03,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:04,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:38:04,165 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:38:04,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:38:04,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:05,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:05,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:05,981 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:08,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:08,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:09,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:10,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:10,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:11,982 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:13,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:13,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:15,073 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:15,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:15,242 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:17,984 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:18,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:18,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:38:19,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:38:19,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:38:19,379 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:38:20,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:20,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:20,380 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:21,986 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:23,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:23,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:25,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:25,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:25,671 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:27,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:27,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:27,988 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:29,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:29,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:31,268 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:32,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:32,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:33,990 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:34,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:38:34,165 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:38:34,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:38:34,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:34,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:36,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:37,992 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:38,105 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:38:38,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:38,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:40,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:40,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:42,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:42,868 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:38:42,869 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:38:42,870 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:38:42,870 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:38:42,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:42,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:42,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:38:43,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:44,995 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:45,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:45,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:47,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:47,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:47,873 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:38:49,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:38:49,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:38:49,394 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:38:50,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:50,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:52,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:52,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:53,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:55,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:55,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:55,998 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:57,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:57,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:59,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:00,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:00,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:02,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:02,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:04,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:39:04,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:39:04,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:39:05,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:05,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:05,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:07,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:07,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:09,879 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:09,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:10,003 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:39:10,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:11,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:11,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:15,285 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:39:15,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:15,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:15,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:17,935 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:17,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:39:19,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:39:19,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:39:19,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:19,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:20,908 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:39:20,908 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:22,008 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:39:22,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:22,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:24,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:24,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:25,910 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:27,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:27,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:29,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:29,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:30,910 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:32,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:32,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:39:34,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:39:34,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:39:34,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:34,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:36,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:39:36,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:37,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:37,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:39,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:39,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:41,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:42,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:42,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:44,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:44,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:46,721 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:46,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:46,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:48,017 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:39:48,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:48,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:39:49,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:39:49,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:39:49,384 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:39:52,385 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:52,439 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:39:52,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:52,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:55,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:55,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:57,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:57,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:57,386 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:00,021 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:00,172 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:00,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:02,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:02,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:03,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:40:04,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:40:04,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:40:05,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:05,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:07,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:07,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:08,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:10,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:10,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:12,025 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:12,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:12,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:14,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:14,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:14,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:17,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:17,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:19,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:40:19,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:40:19,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:40:19,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:19,386 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:40:19,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:19,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:22,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:22,046 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:24,029 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:24,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:24,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:24,757 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:26,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:26,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:29,758 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:29,803 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:40:29,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:29,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:32,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:32,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:40:34,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:40:34,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:40:34,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:34,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:35,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:37,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:37,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:38,033 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:39,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:39,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:40,817 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:42,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:42,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:44,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:44,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:45,818 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:47,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:47,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:49,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:40:49,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:40:49,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:40:49,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:49,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:50,037 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:51,134 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:40:51,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:52,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:52,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:54,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:54,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:56,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:57,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:57,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:59,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:59,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:01,761 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:01,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:01,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:02,041 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:03,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:03,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:41:04,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:41:04,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:41:07,277 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:41:07,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:07,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:07,376 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:10,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:10,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:12,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:12,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:12,377 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:14,045 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:14,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:14,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:16,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:16,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:18,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:19,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:41:19,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:41:19,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:41:19,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:19,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:20,174 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:41:21,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:21,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:23,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:24,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:24,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:26,049 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:26,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:26,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:28,222 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:29,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:29,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:31,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:31,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:33,223 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:41:34,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:41:34,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:41:34,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:34,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:36,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:36,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:38,053 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:38,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:39,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:39,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:41,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:41,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:43,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:44,626 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:41:44,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:44,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:47,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:47,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:48,355 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:49,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:41:49,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:41:49,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:41:49,391 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:41:49,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:49,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:52,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:52,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:52,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:53,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:54,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:54,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:57,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:57,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:58,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:59,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:59,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:02,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:02,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:04,063 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:42:04,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:42:04,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:04,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:42:04,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:04,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:07,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:07,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:09,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:09,076 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:09,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:11,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:11,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:13,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:13,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:14,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:16,066 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:16,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:16,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:18,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:18,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:42:19,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:42:19,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:42:19,392 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:42:19,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:22,098 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:42:22,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:22,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:24,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:24,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:24,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:26,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:26,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:28,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:29,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:29,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:29,954 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:31,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:31,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:42:34,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:42:34,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:42:34,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:34,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:35,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:36,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:36,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:39,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:39,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:40,072 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:41,308 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:41,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:41,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:44,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:44,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:46,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:46,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:46,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:49,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:49,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:49,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:42:49,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:42:49,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:42:50,578 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:42:51,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:51,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:51,579 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:54,076 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:54,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:54,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:56,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:56,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:56,949 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:59,480 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:42:59,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:59,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:01,949 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:02,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:02,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:43:04,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:43:04,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:43:04,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:04,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:06,080 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:07,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:07,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:07,470 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:09,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:09,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:12,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:12,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:12,470 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:14,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:14,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:17,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:17,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:18,083 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:18,094 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:19,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:19,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:43:19,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:43:19,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:43:19,396 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:43:21,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:21,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:23,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:23,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:24,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:26,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:26,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:28,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:28,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:28,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:31,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:31,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:32,087 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:33,629 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:33,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:33,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:43:34,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:43:34,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:43:37,360 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:43:37,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:37,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:39,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:40,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:40,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:42,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:42,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:44,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:44,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:44,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:44,881 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:46,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:46,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:49,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:43:49,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:43:49,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:43:49,397 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:43:49,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:49,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:50,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:51,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:51,771 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:54,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:54,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:56,095 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:56,210 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:56,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:56,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:59,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:59,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:01,210 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:01,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:01,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:44:04,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:44:04,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:44:04,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:04,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:06,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:06,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:06,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:08,099 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:09,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:09,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:11,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:11,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:11,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:14,687 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:44:14,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:14,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:16,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:17,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:17,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:19,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:44:19,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:44:19,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:44:19,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:19,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:20,103 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:21,409 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:44:21,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:22,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:22,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:24,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:24,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:26,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:27,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:27,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:29,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:29,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:32,077 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:32,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:32,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:33,107 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:34,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:44:34,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:44:34,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:44:34,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:34,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:37,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:37,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:37,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:39,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:39,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:42,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:42,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:42,338 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:44,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:44,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:45,110 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:47,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:47,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:47,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:49,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:49,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:44:49,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:44:49,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:44:49,402 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:44:52,353 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:44:52,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:52,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:53,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:55,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:55,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:57,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:57,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:58,738 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:59,115 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:59,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:59,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:01,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:01,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:03,739 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:04,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:45:04,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:45:04,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:45:04,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:04,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:06,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:06,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:09,273 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:09,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:09,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:11,119 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:45:11,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:11,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:14,273 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:14,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:14,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:16,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:16,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:19,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:45:19,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:45:19,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:45:19,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:19,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:19,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:19,403 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:45:21,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:21,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:23,123 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:45:23,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:23,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:24,490 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:26,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:26,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:29,388 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:45:29,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:29,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:29,490 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:32,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:32,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:34,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:34,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:45:34,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:45:34,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:45:35,127 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:45:35,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:37,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:37,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:39,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:39,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:40,356 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:41,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:41,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:43,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:43,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:45,790 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:45,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:45,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:47,131 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:45:48,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:48,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:45:49,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:45:49,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:45:49,404 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:45:50,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:50,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:51,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:53,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:53,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:55,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:55,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:56,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:58,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:58,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:59,136 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:00,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:00,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:01,702 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:03,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:03,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:46:04,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:46:04,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:46:06,739 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:46:06,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:06,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:07,363 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:09,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:09,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:11,139 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:11,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:11,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:12,482 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:14,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:14,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:16,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:16,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:17,482 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:46:19,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:46:19,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:46:19,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:19,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:21,270 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:46:21,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:21,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:22,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:23,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:24,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:24,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:26,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:26,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:27,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:29,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:29,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:31,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:31,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:32,605 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:46:34,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:46:34,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:46:34,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:34,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:36,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:36,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:37,147 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:38,311 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:39,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:39,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:41,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:41,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:43,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:44,567 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:46:44,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:44,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:47,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:47,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:48,596 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:49,150 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:46:49,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:46:49,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:46:49,407 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:46:49,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:49,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:52,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:52,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:54,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:54,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:54,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:57,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:57,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:59,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:59,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:00,093 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:01,154 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:01,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:01,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:04,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:04,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:47:04,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:47:04,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:47:05,324 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:06,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:06,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:08,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:08,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:10,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:11,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:11,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:13,158 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:13,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:13,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:15,904 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:16,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:16,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:18,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:18,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:47:19,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:47:19,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:47:19,409 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:47:21,081 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:21,842 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:47:21,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:21,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:23,161 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:24,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:24,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:26,688 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:27,162 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:27,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:27,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:29,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:30,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:32,277 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:32,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:33,164 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:34,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:47:34,480 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:47:34,481 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:47:37,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:37,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:37,867 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:39,166 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:39,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:39,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:42,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:42,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:42,868 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:44,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:44,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:47,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:47,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:47,869 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:47:49,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:47:49,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:47:49,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:49,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:49,410 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:47:51,170 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:52,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:52,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:53,790 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:54,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:54,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:56,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:56,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:58,790 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:58,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:59,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:01,174 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:02,294 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:48:02,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:02,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:48:04,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:48:04,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:48:04,375 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:05,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:05,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:07,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:07,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:09,375 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:10,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:10,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:12,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:12,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:13,178 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:14,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:14,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:15,035 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:16,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:16,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:19,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:48:19,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:48:19,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:48:19,411 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:48:19,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:19,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:20,964 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:21,181 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:21,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:21,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:24,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:24,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:25,965 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:26,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:26,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:29,183 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:29,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:29,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:31,684 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:31,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:31,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:48:34,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:48:34,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:48:34,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:34,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:36,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:36,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:37,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:39,910 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:48:39,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:39,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:42,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:42,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:43,187 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:43,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:44,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:44,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:47,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:47,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:48,374 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:48:49,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:48:49,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:48:49,413 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:48:49,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:49,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:52,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:52,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:53,191 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:54,078 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:54,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:54,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:57,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:57,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:59,080 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:59,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:59,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:02,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:02,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:49:04,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:49:04,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:49:04,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:04,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:04,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:05,195 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:07,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:07,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:09,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:09,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:09,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:12,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:12,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:14,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:14,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:14,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:15,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:17,474 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:49:17,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:17,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:19,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:49:19,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:49:19,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:49:20,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:20,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:20,540 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:49:20,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:22,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:22,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:25,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:25,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:25,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:27,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:27,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:29,203 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:30,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:30,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:30,990 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:32,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:32,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:49:34,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:49:34,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:49:34,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:34,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:36,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:36,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:36,959 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:37,206 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:39,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:39,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:41,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:41,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:41,960 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:44,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:44,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:45,209 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:46,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:46,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:47,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:49:49,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:49:49,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:49:49,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:49,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:49,416 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:49:51,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:51,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:52,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:54,705 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:49:54,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:54,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:57,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:57,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:58,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:59,214 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:59,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:59,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:02,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:02,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:03,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:50:04,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:50:04,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:50:04,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:04,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:07,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:07,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:08,257 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:09,217 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:09,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:09,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:12,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:12,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:13,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:14,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:14,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:16,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:16,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:18,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:18,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:18,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:50:19,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:50:19,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:50:19,417 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:50:21,221 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:21,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:21,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:23,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:23,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:24,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:26,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:26,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:28,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:28,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:29,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:31,225 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:32,269 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:50:32,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:32,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:33,866 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:34,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:50:34,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:50:34,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:50:35,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:35,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:37,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:37,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:39,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:39,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:39,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:41,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:41,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:43,229 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:44,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:44,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:45,113 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:46,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:46,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:49,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:50:49,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:50:49,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:50:49,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:49,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:50,665 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:50:50,667 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:51,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:51,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:54,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:54,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:55,666 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:56,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:56,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:57,234 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:59,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:59,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:00,758 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:01,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:01,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:04,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:51:04,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:51:04,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:04,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:51:04,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:05,238 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:06,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:06,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:06,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:09,493 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:51:09,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:09,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:11,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:12,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:12,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:14,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:14,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:16,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:17,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:17,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:19,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:19,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:51:19,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:51:19,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:51:19,242 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:20,638 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:51:21,639 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:22,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:22,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:24,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:24,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:26,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:26,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:27,251 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:27,599 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:28,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:29,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:31,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:31,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:32,599 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:33,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:33,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:51:34,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:51:34,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:51:36,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:36,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:38,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:38,652 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:38,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:41,257 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:41,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:41,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:43,454 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:43,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:43,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:46,943 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:51:46,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:46,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:48,455 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:49,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:51:49,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:51:49,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:51:49,260 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:49,423 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:51:49,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:49,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:51,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:51,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:54,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:54,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:54,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:56,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:56,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:59,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:59,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:59,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:01,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:01,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:03,265 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:04,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:52:04,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:52:04,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:52:04,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:04,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:05,335 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:06,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:06,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:09,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:09,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:10,336 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:11,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:11,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:11,268 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:14,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:14,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:15,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:16,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:16,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:18,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:18,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:19,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:52:19,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:52:19,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:52:19,424 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:52:20,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:20,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:21,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:24,225 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:52:24,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:24,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:25,277 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:27,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:27,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:27,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:29,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:29,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:31,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:31,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:32,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:33,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:33,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:33,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:52:34,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:52:34,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:52:36,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:36,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:38,347 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:38,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:38,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:41,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:41,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:43,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:43,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:43,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:46,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:46,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:47,286 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:48,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:48,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:48,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:49,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:52:49,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:52:49,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:52:50,828 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:52:51,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:51,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:53,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:53,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:54,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:55,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:56,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:56,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:58,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:58,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:59,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:01,579 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:53:01,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:01,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:04,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:53:04,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:53:04,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:53:04,309 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:04,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:05,293 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:05,333 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:06,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:06,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:09,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:09,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:10,334 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:11,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:11,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:14,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:14,152 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:15,296 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:15,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:16,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:16,204 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:18,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:18,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:19,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:53:19,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:53:19,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:53:19,428 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:53:21,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:21,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:21,429 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:23,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:23,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:25,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:25,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:26,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:28,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:28,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:29,302 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:30,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:30,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:31,644 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:33,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:33,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:53:34,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:53:34,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:53:35,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:35,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:37,384 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:38,822 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:53:38,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:38,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:39,306 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:41,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:41,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:42,734 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:43,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:43,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:46,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:46,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:47,735 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:48,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:48,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:53:49,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:53:49,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:53:49,430 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:53:51,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:51,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:53,004 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:53,311 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:53,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:53,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:56,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:56,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:58,005 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:58,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:58,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:01,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:01,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:03,006 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:03,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:03,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:04,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:54:04,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:54:04,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:54:05,317 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:06,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:06,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:08,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:08,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:08,628 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:11,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:11,100 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:13,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:13,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:13,629 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:16,410 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:54:16,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:16,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:17,321 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:19,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:19,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:19,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:54:19,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:54:19,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:54:19,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:19,433 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:54:21,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:21,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:23,322 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:23,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:23,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:25,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:25,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:25,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:28,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:28,851 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:30,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:30,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:31,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:31,330 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:33,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:33,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:34,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:54:34,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:54:34,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:54:35,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:35,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:36,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:38,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:38,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:40,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:40,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:41,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:43,334 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:43,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:43,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:45,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:45,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:46,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:48,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:48,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:54:49,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:54:49,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:54:49,433 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:54:50,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:50,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:51,979 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:53,337 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:53,689 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:54:53,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:53,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:56,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:56,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:56,986 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:58,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:58,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:01,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:01,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:01,987 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:03,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:03,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:04,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:55:04,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:55:04,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:55:06,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:06,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:07,341 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:07,822 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:08,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:08,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:11,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:11,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:12,822 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:13,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:13,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:16,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:16,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:17,823 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:18,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:18,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:55:19,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:55:19,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:55:19,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:19,435 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:55:20,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:20,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:22,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:22,956 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:23,436 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:25,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:25,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:27,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:27,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:29,224 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:29,347 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:31,041 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:55:31,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:31,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:33,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:33,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:55:34,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:55:34,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:55:34,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:35,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:35,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:38,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:38,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:39,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:40,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:40,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:41,351 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:43,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:43,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:44,561 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:45,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:45,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:48,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:48,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:49,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:55:49,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:55:49,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:55:49,437 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:55:50,438 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:50,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:50,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:51,354 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:53,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:53,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:55,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:55,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:55,694 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:58,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:58,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:00,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:00,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:00,695 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:03,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:03,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:04,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:56:04,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:56:04,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:56:05,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:05,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:05,359 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:06,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:08,771 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:56:08,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:08,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:10,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:10,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:11,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:13,657 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:13,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:15,689 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:15,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:17,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:17,362 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:17,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:17,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:56:19,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:56:19,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:56:20,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:20,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:20,881 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:56:22,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:22,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:22,882 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:25,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:25,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:27,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:27,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:27,884 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:29,365 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:30,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:30,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:32,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:32,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:33,796 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:56:34,175 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:56:34,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:56:35,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:35,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:37,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:37,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:38,966 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:40,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:40,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:40,368 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:42,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:42,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:44,010 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:45,392 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:56:45,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:45,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:48,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:48,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:49,011 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:49,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:56:49,175 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:56:49,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:56:49,440 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:56:50,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:50,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:53,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:53,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:54,373 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:54,562 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:55,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:55,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:57,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:58,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:59,563 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:00,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:00,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:02,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:02,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:04,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:57:04,176 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:57:04,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:57:04,376 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:04,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:04,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:05,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:07,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:07,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:09,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:09,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:10,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:12,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:12,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:14,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:14,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:15,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:17,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:17,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:18,380 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:57:19,175 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:57:19,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:57:19,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:19,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:20,600 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:57:20,601 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:23,072 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:57:23,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:23,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:25,601 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:26,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:26,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:28,383 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:28,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:28,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:30,759 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:31,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:31,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:33,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:33,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:34,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:57:34,175 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:57:34,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:57:36,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:36,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:36,254 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:57:36,256 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,261 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:57:36,439 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:57:36,439 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:57:37,385 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:57:37,386 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/config.yaml +2024-03-27 20:57:38,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:38,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:38,386 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:41,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:41,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:41,440 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:43,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:43,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:46,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:46,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:46,389 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:47,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:48,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:48,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:49,443 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:57:49,454 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:57:49,454 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:57:49,501 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:57:50,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:50,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:52,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:52,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:53,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:55,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:55,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:57,647 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:58,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:58,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:02,447 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:58:02,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:02,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:02,648 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:04,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:58:04,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:58:04,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:58:05,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:05,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:07,825 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:08,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:08,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:08,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/config.yaml +2024-03-27 20:58:08,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:10,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:10,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:12,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:12,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:13,284 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:13,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:15,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:15,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:16,397 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:18,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:18,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:18,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:19,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:58:19,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:58:19,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:58:19,573 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:58:20,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:20,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:22,399 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:23,367 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:23,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:23,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:25,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:25,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:28,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:28,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:28,401 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:28,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:30,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:30,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:33,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:33,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:33,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:34,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:34,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:58:34,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:58:34,427 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:58:35,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:35,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:38,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:38,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:38,556 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:40,405 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:41,976 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:58:41,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:42,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:44,465 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:45,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:45,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:46,406 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:47,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:47,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:49,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:58:49,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:58:49,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:58:49,551 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:58:49,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:50,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:50,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:52,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:52,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:52,409 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:55,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:55,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:55,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:57,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:57,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:58,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:59:00,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:00,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:00,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:02,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:02,217 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:04,415 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:59:04,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:59:04,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:59:04,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:59:05,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:05,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:05,633 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:07,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:07,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:10,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:10,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:10,416 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:59:11,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:12,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:12,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:15,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:15,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:16,418 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:59:16,640 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:17,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:17,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:19,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:59:19,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:59:19,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:59:20,576 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:59:20,834 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:59:20,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:20,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:21,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:22,420 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log