diff --git "a/wandb/run-20240327_123544-5vwxt2ut/logs/debug-internal.log" "b/wandb/run-20240327_123544-5vwxt2ut/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240327_123544-5vwxt2ut/logs/debug-internal.log" @@ -0,0 +1,1807 @@ +2024-03-27 12:35:44,067 INFO StreamThr :1368934 [internal.py:wandb_internal():86] W&B internal server running at pid: 1368934, started at: 2024-03-27 12:35:44.067171 +2024-03-27 12:35:44,070 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status +2024-03-27 12:35:44,072 INFO WriterThread:1368934 [datastore.py:open_for_write():87] open: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/run-5vwxt2ut.wandb +2024-03-27 12:35:44,073 DEBUG SenderThread:1368934 [sender.py:send():382] send: header +2024-03-27 12:35:44,240 DEBUG SenderThread:1368934 [sender.py:send():382] send: run +2024-03-27 12:35:44,583 INFO SenderThread:1368934 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files +2024-03-27 12:35:44,584 INFO SenderThread:1368934 [sender.py:_start_run_threads():1136] run started: 5vwxt2ut with start time 1711539344.066994 +2024-03-27 12:35:44,589 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: check_version +2024-03-27 12:35:44,590 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: check_version +2024-03-27 12:35:44,632 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: run_start +2024-03-27 12:35:44,644 DEBUG HandlerThread:1368934 [system_info.py:__init__():27] System info init +2024-03-27 12:35:44,644 DEBUG HandlerThread:1368934 [system_info.py:__init__():42] System info init done +2024-03-27 12:35:44,644 INFO HandlerThread:1368934 [system_monitor.py:start():194] Starting system monitor +2024-03-27 12:35:44,644 INFO SystemMonitor:1368934 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-27 12:35:44,645 INFO HandlerThread:1368934 [system_monitor.py:probe():214] Collecting system info +2024-03-27 12:35:44,645 INFO SystemMonitor:1368934 [interfaces.py:start():190] Started cpu monitoring +2024-03-27 12:35:44,646 INFO SystemMonitor:1368934 [interfaces.py:start():190] Started disk monitoring +2024-03-27 12:35:44,647 INFO SystemMonitor:1368934 [interfaces.py:start():190] Started gpu monitoring +2024-03-27 12:35:44,650 INFO SystemMonitor:1368934 [interfaces.py:start():190] Started memory monitoring +2024-03-27 12:35:44,651 INFO SystemMonitor:1368934 [interfaces.py:start():190] Started network monitoring +2024-03-27 12:35:44,688 DEBUG HandlerThread:1368934 [system_info.py:probe():151] Probing system +2024-03-27 12:35:44,694 DEBUG HandlerThread:1368934 [system_info.py:_probe_git():136] Probing git +2024-03-27 12:35:44,703 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:44,704 DEBUG HandlerThread:1368934 [system_info.py:_probe_git():144] Probing git done +2024-03-27 12:35:44,705 DEBUG HandlerThread:1368934 [system_info.py:probe():199] Probing system done +2024-03-27 12:35:44,705 DEBUG HandlerThread:1368934 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-03-27T11:35:44.688111', 'startedAt': '2024-03-27T11:35:44.063111', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=distil-whisper/distil-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_1', '--dataset_config_name=hi', '--language=hindi', '--train_split_name=train+validation', '--eval_split_name=test', '--max_steps=5000', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=16', '--logging_steps=25', '--learning_rate=1e-5', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--generation_max_length=225', '--preprocessing_num_workers=1', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--gradient_checkpointing', '--group_by_length', '--fp16', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-large-v3-hi-ft', 'commit': '40c686df113c0e98e7363c1bd523f58d11848fc0'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/distil-large-v3-hi-ft', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2320.8891093750008, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 1720.022, 'min': 1500.0, 'max': 2250.0}, {'current': 3381.801, 'min': 1500.0, 'max': 2250.0}, {'current': 1739.128, 'min': 1500.0, 'max': 2250.0}, {'current': 3391.513, 'min': 1500.0, 'max': 2250.0}, {'current': 3157.808, 'min': 1500.0, 'max': 2250.0}, {'current': 1901.238, 'min': 1500.0, 'max': 2250.0}, {'current': 2217.668, 'min': 1500.0, 'max': 2250.0}, {'current': 1915.314, 'min': 1500.0, 'max': 2250.0}, {'current': 1752.905, 'min': 1500.0, 'max': 2250.0}, {'current': 2292.065, 'min': 1500.0, 'max': 2250.0}, {'current': 1948.336, 'min': 1500.0, 'max': 2250.0}, {'current': 1751.862, 'min': 1500.0, 'max': 2250.0}, {'current': 1919.808, 'min': 1500.0, 'max': 2250.0}, {'current': 1754.656, 'min': 1500.0, 'max': 2250.0}, {'current': 2313.669, 'min': 1500.0, 'max': 2250.0}, {'current': 1811.724, 'min': 1500.0, 'max': 2250.0}, {'current': 1807.199, 'min': 1500.0, 'max': 2250.0}, {'current': 1692.815, 'min': 1500.0, 'max': 2250.0}, {'current': 1842.85, 'min': 1500.0, 'max': 2250.0}, {'current': 1893.814, 'min': 1500.0, 'max': 2250.0}, {'current': 1925.522, 'min': 1500.0, 'max': 2250.0}, {'current': 1756.882, 'min': 1500.0, 'max': 2250.0}, {'current': 2014.845, 'min': 1500.0, 'max': 2250.0}, {'current': 1937.037, 'min': 1500.0, 'max': 2250.0}, {'current': 1938.273, 'min': 1500.0, 'max': 2250.0}, {'current': 2292.805, 'min': 1500.0, 'max': 2250.0}, {'current': 1753.629, 'min': 1500.0, 'max': 2250.0}, {'current': 1753.05, 'min': 1500.0, 'max': 2250.0}, {'current': 1944.417, 'min': 1500.0, 'max': 2250.0}, {'current': 1754.514, 'min': 1500.0, 'max': 2250.0}, {'current': 2280.397, 'min': 1500.0, 'max': 2250.0}, {'current': 1750.289, 'min': 1500.0, 'max': 2250.0}, {'current': 1751.461, 'min': 1500.0, 'max': 2250.0}, {'current': 1753.829, 'min': 1500.0, 'max': 2250.0}, {'current': 2702.274, 'min': 1500.0, 'max': 2250.0}, {'current': 1921.157, 'min': 1500.0, 'max': 2250.0}, {'current': 1741.841, 'min': 1500.0, 'max': 2250.0}, {'current': 2111.679, 'min': 1500.0, 'max': 2250.0}, {'current': 1708.371, 'min': 1500.0, 'max': 2250.0}, {'current': 3387.844, 'min': 1500.0, 'max': 2250.0}, {'current': 1696.851, 'min': 1500.0, 'max': 2250.0}, {'current': 2832.842, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.055, 'min': 1500.0, 'max': 2250.0}, {'current': 2207.381, 'min': 1500.0, 'max': 2250.0}, {'current': 1755.308, 'min': 1500.0, 'max': 2250.0}, {'current': 1754.396, 'min': 1500.0, 'max': 2250.0}, {'current': 1932.895, 'min': 1500.0, 'max': 2250.0}, {'current': 1752.406, 'min': 1500.0, 'max': 2250.0}, {'current': 1868.451, 'min': 1500.0, 'max': 2250.0}, {'current': 2009.47, 'min': 1500.0, 'max': 2250.0}, {'current': 1693.917, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.149, 'min': 1500.0, 'max': 2250.0}, {'current': 2241.039, 'min': 1500.0, 'max': 2250.0}, {'current': 2218.22, 'min': 1500.0, 'max': 2250.0}, {'current': 2166.763, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.42, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.263, 'min': 1500.0, 'max': 2250.0}, {'current': 2342.293, 'min': 1500.0, 'max': 2250.0}, {'current': 1773.941, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.693, 'min': 1500.0, 'max': 2250.0}, {'current': 2299.762, 'min': 1500.0, 'max': 2250.0}, {'current': 1929.979, 'min': 1500.0, 'max': 2250.0}, {'current': 2270.019, 'min': 1500.0, 'max': 2250.0}, {'current': 1947.148, 'min': 1500.0, 'max': 2250.0}, {'current': 2032.551, 'min': 1500.0, 'max': 2250.0}, {'current': 1742.768, 'min': 1500.0, 'max': 2250.0}, {'current': 1960.724, 'min': 1500.0, 'max': 2250.0}, {'current': 3357.093, 'min': 1500.0, 'max': 2250.0}, {'current': 3162.732, 'min': 1500.0, 'max': 2250.0}, {'current': 2133.692, 'min': 1500.0, 'max': 2250.0}, {'current': 2005.547, 'min': 1500.0, 'max': 2250.0}, {'current': 1860.04, 'min': 1500.0, 'max': 2250.0}, {'current': 2274.058, 'min': 1500.0, 'max': 2250.0}, {'current': 1883.804, 'min': 1500.0, 'max': 2250.0}, {'current': 2102.025, 'min': 1500.0, 'max': 2250.0}, {'current': 3038.986, 'min': 1500.0, 'max': 2250.0}, {'current': 1850.937, 'min': 1500.0, 'max': 2250.0}, {'current': 2286.556, 'min': 1500.0, 'max': 2250.0}, {'current': 2093.112, 'min': 1500.0, 'max': 2250.0}, {'current': 3080.388, 'min': 1500.0, 'max': 2250.0}, {'current': 3014.044, 'min': 1500.0, 'max': 2250.0}, {'current': 3067.336, 'min': 1500.0, 'max': 2250.0}, {'current': 2955.438, 'min': 1500.0, 'max': 2250.0}, {'current': 2070.708, 'min': 1500.0, 'max': 2250.0}, {'current': 3033.019, 'min': 1500.0, 'max': 2250.0}, {'current': 3016.806, 'min': 1500.0, 'max': 2250.0}, {'current': 2086.652, 'min': 1500.0, 'max': 2250.0}, {'current': 3038.198, 'min': 1500.0, 'max': 2250.0}, {'current': 3022.183, 'min': 1500.0, 'max': 2250.0}, {'current': 2151.059, 'min': 1500.0, 'max': 2250.0}, {'current': 3362.174, 'min': 1500.0, 'max': 2250.0}, {'current': 3382.004, 'min': 1500.0, 'max': 2250.0}, {'current': 3369.6, 'min': 1500.0, 'max': 2250.0}, {'current': 3367.906, 'min': 1500.0, 'max': 2250.0}, {'current': 2176.994, 'min': 1500.0, 'max': 2250.0}, {'current': 3368.516, 'min': 1500.0, 'max': 2250.0}, {'current': 3358.119, 'min': 1500.0, 'max': 2250.0}, {'current': 2422.973, 'min': 1500.0, 'max': 2250.0}, {'current': 1943.984, 'min': 1500.0, 'max': 2250.0}, {'current': 1681.849, 'min': 1500.0, 'max': 2250.0}, {'current': 1630.654, 'min': 1500.0, 'max': 2250.0}, {'current': 1691.891, 'min': 1500.0, 'max': 2250.0}, {'current': 1691.917, 'min': 1500.0, 'max': 2250.0}, {'current': 3265.59, 'min': 1500.0, 'max': 2250.0}, {'current': 1692.935, 'min': 1500.0, 'max': 2250.0}, {'current': 3368.811, 'min': 1500.0, 'max': 2250.0}, {'current': 1687.014, 'min': 1500.0, 'max': 2250.0}, {'current': 2499.16, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.677, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.963, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.49, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.159, 'min': 1500.0, 'max': 2250.0}, {'current': 1845.387, 'min': 1500.0, 'max': 2250.0}, {'current': 2385.622, 'min': 1500.0, 'max': 2250.0}, {'current': 1683.197, 'min': 1500.0, 'max': 2250.0}, {'current': 1683.711, 'min': 1500.0, 'max': 2250.0}, {'current': 1777.23, 'min': 1500.0, 'max': 2250.0}, {'current': 1778.423, 'min': 1500.0, 'max': 2250.0}, {'current': 1916.687, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.278, 'min': 1500.0, 'max': 2250.0}, {'current': 1776.15, 'min': 1500.0, 'max': 2250.0}, {'current': 2028.709, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.557, 'min': 1500.0, 'max': 2250.0}, {'current': 1777.429, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.342, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.553, 'min': 1500.0, 'max': 2250.0}, {'current': 1796.501, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.512, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1499.6025924682617}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-03-27 12:35:44,705 INFO HandlerThread:1368934 [system_monitor.py:probe():224] Finished collecting system info +2024-03-27 12:35:44,705 INFO HandlerThread:1368934 [system_monitor.py:probe():227] Publishing system info +2024-03-27 12:35:44,706 INFO HandlerThread:1368934 [system_monitor.py:probe():229] Finished publishing system info +2024-03-27 12:35:44,710 DEBUG SenderThread:1368934 [sender.py:send():382] send: files +2024-03-27 12:35:44,711 INFO SenderThread:1368934 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-27 12:35:44,715 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: python_packages +2024-03-27 12:35:44,716 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: python_packages +2024-03-27 12:35:44,716 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:35:44,717 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:35:44,719 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:44,719 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:35:44,864 DEBUG SenderThread:1368934 [sender.py:send():382] send: telemetry +2024-03-27 12:35:44,864 DEBUG SenderThread:1368934 [sender.py:send():382] send: config +2024-03-27 12:35:44,865 DEBUG SenderThread:1368934 [sender.py:send():382] send: metric +2024-03-27 12:35:44,865 DEBUG SenderThread:1368934 [sender.py:send():382] send: telemetry +2024-03-27 12:35:44,865 DEBUG SenderThread:1368934 [sender.py:send():382] send: metric +2024-03-27 12:35:44,865 WARNING SenderThread:1368934 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-27 12:35:44,865 DEBUG SenderThread:1368934 [sender.py:send():382] send: telemetry +2024-03-27 12:35:45,218 INFO wandb-upload_0:1368934 [upload_job.py:push():131] Uploaded file /tmp/tmpjdlzrd1zwandb/ijm44w45-wandb-metadata.json +2024-03-27 12:35:45,585 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-metadata.json +2024-03-27 12:35:45,585 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:35:45,585 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/requirements.txt +2024-03-27 12:35:47,584 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:35:47,588 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:47,598 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:49,615 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:49,623 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:49,866 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:35:52,172 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:52,180 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:54,202 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:54,212 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:54,867 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:35:56,590 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:56,601 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:58,623 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:58,635 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:35:59,587 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:35:59,716 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:35:59,716 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:35:59,719 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:35:59,889 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:01,437 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:01,446 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:03,463 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:03,479 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:03,588 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:36:05,514 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:05,855 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:05,864 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:07,899 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:07,932 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:10,603 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:10,613 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:11,003 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:11,590 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:36:12,635 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:12,644 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:15,185 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:15,194 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:15,260 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:36:15,260 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:36:15,322 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:36:16,403 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:17,214 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:17,223 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:17,605 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/config.yaml +2024-03-27 12:36:19,606 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:36:20,478 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:36:20,495 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:20,505 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:22,403 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:23,037 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:23,048 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:25,075 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:25,086 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:27,608 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:36:27,734 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:27,743 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:27,840 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:29,763 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:29,773 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:31,210 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:36:31,978 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:36:31,978 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:36:32,156 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:32,165 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:33,167 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:34,188 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:34,204 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:37,039 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:37,050 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:37,610 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:36:39,068 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:39,081 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:39,158 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:41,868 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:41,884 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:43,905 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:43,914 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:44,500 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:44,651 DEBUG SystemMonitor:1368934 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-27 12:36:44,653 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:36:45,612 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:36:46,571 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:46,580 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:48,601 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:48,611 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:48,787 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:36:48,788 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:36:48,831 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:36:50,932 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:51,176 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:51,187 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:53,205 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:53,213 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:53,614 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:36:56,006 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:36:56,244 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:36:56,265 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:56,275 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:59,001 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:36:59,011 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:01,032 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:01,042 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:01,310 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:01,616 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:37:03,800 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:03,810 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:04,748 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:37:04,748 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:37:05,561 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:37:05,828 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:05,836 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:06,534 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:08,787 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:08,797 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:09,618 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:37:10,817 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:10,827 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:11,734 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:13,243 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:13,252 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:14,655 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:37:15,280 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:15,294 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:16,279 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:37:16,279 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:37:17,075 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:17,317 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:17,330 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:19,621 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:37:19,762 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:37:19,917 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:19,927 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:21,948 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:21,960 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:22,623 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:24,524 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:24,536 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:26,553 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:26,562 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:27,623 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:37:28,454 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:29,137 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:29,146 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:31,279 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:37:31,279 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:37:32,800 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:37:32,819 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:32,833 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:33,882 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:37:33,978 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:35,571 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:35,581 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:35,626 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:37:37,598 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:37,607 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:38,979 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:40,312 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:40,322 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:42,347 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:42,357 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:43,628 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:37:44,379 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:44,657 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:37:44,911 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:44,921 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:46,939 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:46,948 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:47,598 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:37:47,598 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:37:48,882 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:37:49,635 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:49,645 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:50,025 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:51,630 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:37:51,662 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:51,670 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:54,284 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:54,293 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:55,718 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:37:56,313 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:56,322 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:59,184 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:37:59,194 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:01,114 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:01,210 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:01,222 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:01,633 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:38:03,858 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:03,868 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:04,298 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:38:04,299 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:38:04,327 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:38:05,889 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:05,906 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:06,528 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:09,158 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:38:09,178 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:09,186 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:09,636 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:38:11,595 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:11,604 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:12,515 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:13,626 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:13,636 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:14,658 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:38:15,660 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:15,671 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:17,638 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:38:17,854 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:18,222 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:18,231 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:20,254 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:20,264 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:20,270 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:38:20,271 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:38:20,962 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:38:23,076 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:23,097 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:23,109 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:25,128 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:25,141 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:25,640 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:38:27,713 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:27,723 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:28,236 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:29,738 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:29,748 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:31,812 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:38:31,812 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:38:32,696 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:32,708 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:33,634 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:33,642 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:38:34,725 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:34,733 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:37,342 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:37,352 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:37,726 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:38:38,781 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:39,379 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:39,392 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:41,644 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:38:42,007 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:42,018 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:43,991 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:44,660 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:38:46,594 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:38:46,625 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:46,639 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:46,812 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:38:46,812 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:38:48,657 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:48,668 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:50,394 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:50,688 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:51,565 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:51,647 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:38:52,059 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:38:53,588 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:53,598 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:55,504 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:38:56,156 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:56,166 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:58,183 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:58,193 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:38:59,650 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:00,733 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:00,742 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:00,765 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:02,758 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:02,768 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:03,026 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:39:03,026 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:39:05,352 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:39:05,422 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:05,433 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:06,102 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:07,452 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:07,469 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:07,652 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:09,940 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:09,949 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:11,513 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:11,976 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:11,986 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:14,604 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:14,620 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:14,662 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:39:15,654 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:16,663 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:17,196 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:17,208 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:19,228 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:19,237 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:19,664 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:39:19,664 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:39:20,352 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:39:21,867 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:22,542 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:39:22,569 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:22,580 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:22,905 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 12:39:22,907 DEBUG SenderThread:1368934 [sender.py:send():382] send: metric +2024-03-27 12:39:22,907 DEBUG SenderThread:1368934 [sender.py:send():382] send: metric +2024-03-27 12:39:22,907 DEBUG SenderThread:1368934 [sender.py:send():382] send: metric +2024-03-27 12:39:22,907 DEBUG SenderThread:1368934 [sender.py:send():382] send: metric +2024-03-27 12:39:22,907 DEBUG SenderThread:1368934 [sender.py:send():382] send: history +2024-03-27 12:39:22,907 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: summary_record +2024-03-27 12:39:22,909 INFO SenderThread:1368934 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 12:39:23,656 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:23,656 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json +2024-03-27 12:39:25,032 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:25,041 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:25,657 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:26,911 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:27,059 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:27,069 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:29,615 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:29,625 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:31,641 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:31,652 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:32,186 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:32,659 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/config.yaml +2024-03-27 12:39:33,659 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:34,196 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:34,205 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:35,594 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:39:35,594 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:39:36,224 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:36,232 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:36,411 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:39:38,053 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:39,036 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:39,046 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:41,066 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:41,075 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:41,661 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:43,492 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:43,502 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:43,562 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:44,664 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:39:45,525 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:45,541 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:48,449 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:48,469 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:48,898 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:49,663 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:49,701 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:39:49,701 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:39:50,490 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:50,499 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:52,866 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:52,876 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:53,067 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:39:53,984 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:39:54,894 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:54,904 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:57,666 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:39:58,144 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:39:58,163 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:58,173 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:39:59,220 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:00,660 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:00,671 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:02,707 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:02,722 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:03,843 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:40:03,843 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:40:04,535 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:05,359 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:05,369 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:05,668 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:40:07,387 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:07,398 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:09,536 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:09,694 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:40:10,066 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:10,078 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:12,098 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:12,116 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:13,671 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:40:14,138 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:14,147 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:14,665 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:40:14,667 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:16,691 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:16,700 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:18,725 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:18,735 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:18,844 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:40:18,844 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:40:20,043 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:21,723 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:21,735 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:23,674 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:40:23,755 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:23,766 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:25,138 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:26,270 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:26,280 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:26,587 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:40:28,297 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:28,310 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:30,798 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:31,020 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:31,029 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:31,677 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:40:34,420 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:40:34,436 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:34,445 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:34,987 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:40:34,987 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:40:36,153 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:37,074 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:37,084 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:37,388 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:40:39,101 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:39,110 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:39,679 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:40:41,197 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:41,996 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:42,005 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:44,028 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:44,042 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:44,667 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:40:46,554 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:46,767 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:46,781 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:47,682 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:40:48,805 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:48,815 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:51,407 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:51,419 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:51,555 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:51,687 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:40:51,687 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:40:52,388 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:40:53,441 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:53,450 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:55,684 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:40:56,249 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:56,258 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:56,953 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:40:58,277 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:40:58,288 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:00,672 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:00,682 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:01,954 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:02,701 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:02,715 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:05,381 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:05,392 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:05,687 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:41:07,313 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:07,414 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:07,425 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:08,418 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:41:08,418 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:41:08,472 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:41:10,700 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:41:10,721 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:10,732 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:12,613 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:13,312 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:13,326 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:13,690 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:41:14,668 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:41:15,345 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:15,356 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:17,670 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:18,140 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:18,163 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:20,182 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:20,193 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:21,693 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:41:22,850 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:22,861 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:23,012 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:24,882 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:24,891 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:25,131 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:41:25,131 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:41:25,242 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:41:27,556 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:27,572 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:28,290 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:29,592 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:29,608 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:29,696 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:41:32,122 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:32,132 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:33,394 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:34,154 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:34,165 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:35,969 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:41:35,970 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:41:37,016 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:37,024 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:37,698 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:41:38,778 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:39,048 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:39,060 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:41,636 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:41,645 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:41,938 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:41:43,669 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:43,678 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:43,898 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:44,670 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:41:45,702 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:41:47,027 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:41:47,044 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:47,056 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:49,137 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:49,583 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:49,597 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:50,970 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:41:50,970 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:41:51,644 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:51,660 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:54,428 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:54,439 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:54,555 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:41:55,705 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:41:56,466 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:56,482 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:57,975 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:41:58,983 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:41:58,993 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:00,215 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:01,013 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:01,022 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:03,708 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:42:03,992 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:04,001 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:05,915 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:06,023 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:06,042 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:07,028 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:42:07,028 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:42:08,783 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:08,795 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:09,496 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:42:10,822 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:10,835 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:11,231 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:11,710 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:42:13,609 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:13,618 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:14,671 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:42:15,638 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:15,647 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:16,673 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:18,425 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:18,438 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:19,713 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:42:20,457 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:20,467 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:22,570 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:23,471 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:42:23,495 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:23,505 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:23,672 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:42:23,673 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:42:24,496 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:42:26,169 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:26,179 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:27,716 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:42:27,908 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:28,200 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:28,209 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:30,605 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:30,617 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:32,635 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:32,650 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:32,908 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:35,712 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:35,718 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:42:35,723 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:37,743 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:37,752 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:38,270 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:40,230 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:40,240 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:40,468 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:42:40,469 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:42:40,510 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:42:42,263 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:42,274 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:43,572 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:44,290 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:44,313 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:44,673 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:42:45,721 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:42:46,828 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:46,838 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:48,675 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:48,855 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:48,865 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:51,637 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:51,648 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:51,919 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 12:42:51,920 DEBUG SenderThread:1368934 [sender.py:send():382] send: history +2024-03-27 12:42:51,921 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: summary_record +2024-03-27 12:42:51,921 INFO SenderThread:1368934 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 12:42:52,723 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json +2024-03-27 12:42:53,667 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:53,681 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:53,723 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:42:53,922 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:56,111 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:56,120 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:56,990 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:42:57,099 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:42:57,099 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:42:59,296 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:42:59,860 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:42:59,877 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:42:59,889 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:01,726 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:43:02,388 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:02,400 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:04,418 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:04,426 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:05,213 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:07,351 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:07,384 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:09,404 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:09,413 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:09,728 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:43:10,674 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:12,038 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:12,049 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:12,192 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:43:12,192 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:43:13,072 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:43:14,080 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:14,090 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:14,675 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:43:15,676 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:16,855 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:16,866 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:17,731 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:43:18,885 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:18,893 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:21,095 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:21,380 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:21,389 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:23,406 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:23,416 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:24,613 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:43:24,613 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:43:26,138 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:26,149 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:26,456 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:27,278 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:43:27,734 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:43:28,166 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:28,175 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:30,828 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:30,837 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:32,018 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:32,857 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:32,872 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:35,736 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:43:36,112 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:43:36,131 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:36,143 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:37,874 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:38,720 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:38,729 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:39,613 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:43:39,614 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:43:40,748 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:40,762 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:41,412 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:43:43,154 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:43,525 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:43,534 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:43,738 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:43:44,677 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:43:45,553 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:45,568 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:48,124 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:48,134 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:48,584 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:50,159 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:50,168 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:51,741 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:43:52,844 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:52,854 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:54,556 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:43:54,877 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:54,886 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:55,697 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:43:55,697 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:43:56,412 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:43:57,490 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:57,503 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:59,519 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:59,529 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:43:59,743 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:43:59,900 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:02,168 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:02,177 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:04,193 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:04,202 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:04,901 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:07,108 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:07,121 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:07,745 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:44:09,137 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:09,145 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:10,279 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:12,297 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:44:12,314 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:12,325 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:12,381 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:44:12,382 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:44:12,470 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:44:14,678 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:44:15,037 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:15,049 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:15,656 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:17,066 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:17,078 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:17,748 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:44:19,660 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:19,670 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:20,657 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:21,688 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:21,697 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:24,713 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:24,725 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:25,750 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:44:26,074 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:26,736 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:44:26,737 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:44:26,744 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:26,754 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:29,247 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:44:29,452 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:29,463 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:31,361 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:31,491 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:31,501 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:33,753 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:44:34,372 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:34,382 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:36,405 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:36,414 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:36,433 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:38,968 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:38,980 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:40,020 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:44:40,021 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:44:41,000 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:41,009 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:41,755 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:44:41,892 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:43,802 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:43,812 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:44,680 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:44:45,828 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:45,838 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:46,012 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:44:46,968 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:49,230 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:44:49,263 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:49,283 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:49,758 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:44:51,657 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:51,666 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:52,232 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:53,683 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:53,693 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:55,020 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:44:55,021 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:44:56,428 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:56,438 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:56,842 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:44:57,512 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:44:58,462 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:58,471 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:44:59,761 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:45:01,252 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:01,261 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:03,011 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:03,289 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:03,300 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:06,272 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:06,284 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:07,763 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:45:08,304 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:08,314 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:08,833 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:10,949 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:10,961 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:11,151 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:45:11,152 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:45:11,842 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:45:12,979 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:12,988 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:14,151 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:14,681 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:45:15,765 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:45:15,945 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:15,957 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:17,973 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:17,981 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:19,492 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:20,745 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:20,756 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:22,774 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:22,784 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:23,768 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:45:25,430 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:25,972 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:45:25,990 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:26,001 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:27,614 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:45:27,712 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:45:27,712 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:45:28,635 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:28,658 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:30,678 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:30,688 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:30,767 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:31,770 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:45:33,307 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:33,320 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:35,335 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:35,344 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:35,768 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:37,970 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:37,984 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:39,773 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:45:40,001 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:40,009 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:41,169 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:42,544 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:42,554 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:44,208 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:45:44,237 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:45:44,238 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:45:44,571 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:44,580 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:44,683 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:45:47,082 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:47,518 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:47,528 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:47,775 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:45:49,552 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:49,561 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:52,044 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:52,054 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:52,447 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:54,093 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:54,102 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:57,029 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:57,038 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:57,778 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:45:57,789 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:45:58,454 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:45:58,454 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:45:59,057 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:45:59,067 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:00,850 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:46:02,180 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:46:02,203 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:02,217 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:02,881 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:04,983 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:04,992 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:05,780 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:46:07,009 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:07,021 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:08,113 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:09,412 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:09,421 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:11,446 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:11,456 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:11,718 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:46:11,718 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:46:13,389 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:13,782 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:46:14,435 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:14,444 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:14,684 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:46:16,468 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:16,477 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:17,514 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:46:18,543 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:19,031 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:19,042 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:20,755 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 12:46:20,757 DEBUG SenderThread:1368934 [sender.py:send():382] send: history +2024-03-27 12:46:20,757 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: summary_record +2024-03-27 12:46:20,758 INFO SenderThread:1368934 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 12:46:20,784 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json +2024-03-27 12:46:21,059 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:21,075 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:21,784 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:46:23,758 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:23,760 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:23,770 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:25,843 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:25,859 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:26,718 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:46:26,718 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:46:28,892 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:28,986 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:29,002 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:29,786 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:46:31,026 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:31,036 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:32,506 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:46:33,638 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:33,648 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:34,040 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:35,667 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:35,679 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:37,789 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:46:39,022 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:46:39,048 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:39,058 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:39,332 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:41,665 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:41,676 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:42,339 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:46:42,340 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:46:43,698 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:43,721 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:44,533 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:44,686 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:46:45,001 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:46:45,791 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:46:46,578 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:46,590 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:48,607 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:48,618 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:49,570 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:51,109 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:51,124 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:53,150 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:53,161 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:54,871 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:46:55,794 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:46:56,131 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:56,141 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:58,160 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:58,169 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:46:58,905 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:46:58,905 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:47:00,001 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:47:00,132 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:00,749 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:00,762 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:02,778 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:02,787 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:03,796 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:47:05,174 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:05,549 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:05,560 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:07,577 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:07,586 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:10,174 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:10,467 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:10,479 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:11,798 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:47:12,498 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:12,509 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:14,688 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:47:15,585 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:47:15,591 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:47:15,592 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:47:15,603 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:15,614 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:15,689 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:47:15,766 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:18,317 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:18,327 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:19,800 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:47:20,346 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:20,355 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:20,853 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:23,043 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:23,053 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:25,096 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:25,128 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:25,853 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:27,802 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:47:27,869 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:27,877 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:29,757 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:47:29,757 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:47:29,897 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:29,907 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:30,979 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:32,229 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:47:32,491 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:32,500 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:34,524 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:34,533 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:35,805 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:47:36,388 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:37,510 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:37,521 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:39,541 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:39,550 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:41,388 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:42,110 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:42,120 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:44,139 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:44,148 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:44,689 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:47:46,691 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:46,807 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:47:46,947 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:46,959 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:48,903 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:47:48,903 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:47:48,944 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:47:48,977 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:48,987 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:52,123 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:52,337 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:47:52,354 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:52,362 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:54,773 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:54,781 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:54,810 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:47:56,799 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:56,809 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:57,124 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:47:59,562 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:59,580 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:47:59,759 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:47:59,759 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:48:01,598 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:01,607 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:02,479 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:02,812 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:48:04,161 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:04,171 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:04,837 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:48:06,196 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:06,206 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:07,560 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:09,145 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:09,156 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:10,815 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:48:11,174 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:11,184 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:12,803 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:13,777 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:13,787 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:14,691 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:48:14,759 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:48:14,759 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:48:15,814 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:15,824 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:18,104 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:18,673 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:18,682 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:18,817 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:48:20,701 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:20,710 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:22,275 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:48:23,112 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:23,122 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:23,666 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:25,149 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:25,160 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:26,819 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:48:28,537 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:48:28,568 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:28,579 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:29,482 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:30,641 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:48:30,641 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:48:31,281 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:31,291 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:33,311 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:33,327 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:34,861 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:36,095 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:36,104 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:36,822 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:48:38,122 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:38,130 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:38,269 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:48:40,023 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:40,795 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:40,807 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:42,833 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:42,841 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:44,693 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:48:44,824 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:48:45,459 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:45,468 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:45,694 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:46,590 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:48:46,591 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:48:47,484 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:47,494 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:49,814 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:48:50,266 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:50,276 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:51,536 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:52,294 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:52,305 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:52,827 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:48:54,710 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:54,719 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:56,537 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:48:56,737 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:56,746 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:59,704 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:48:59,714 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:00,829 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:01,736 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:01,747 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:01,934 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:02,495 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:49:02,495 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:49:04,785 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:49:04,810 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:04,814 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:49:04,819 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:07,312 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:07,612 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:07,624 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:08,831 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:09,642 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:09,651 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:12,209 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:12,220 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:12,313 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:14,243 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:14,253 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:14,694 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:49:16,834 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:17,100 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:17,113 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:17,713 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:18,381 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:49:18,381 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:49:19,138 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:19,148 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:20,819 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:49:21,627 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:21,639 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:22,778 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:23,660 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:23,678 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:26,462 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:26,473 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:26,836 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:28,011 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:28,499 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:28,509 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:31,279 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:31,292 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:31,642 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:49:31,643 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:49:33,314 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:33,323 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:33,328 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:34,839 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:36,087 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:36,096 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:37,429 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:49:38,122 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:38,132 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:38,402 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:41,624 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:49:41,644 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:41,653 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:42,841 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:43,628 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:44,172 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:44,183 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:44,696 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:49:46,211 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:46,220 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:46,642 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:49:46,643 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:49:48,778 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:49,016 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:49,026 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 12:49:49,027 DEBUG SenderThread:1368934 [sender.py:send():382] send: history +2024-03-27 12:49:49,027 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: summary_record +2024-03-27 12:49:49,028 INFO SenderThread:1368934 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 12:49:49,032 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:49,843 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:49,843 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json +2024-03-27 12:49:50,843 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:51,051 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:51,060 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:53,650 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:53,659 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:54,020 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:49:54,029 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:49:55,685 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:55,695 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:58,535 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:58,545 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:49:58,846 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:49:59,316 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:00,563 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:00,574 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:02,463 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:50:02,464 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:50:03,165 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:03,175 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:04,628 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:05,193 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:05,204 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:06,848 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:50:08,003 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:08,015 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:09,678 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:10,032 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:10,037 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:50:10,042 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:12,815 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:12,828 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:14,697 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:50:14,699 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:14,853 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:14,865 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:16,851 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:50:17,705 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:50:17,705 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:50:18,054 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:50:18,077 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:18,090 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:19,854 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:20,581 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:20,591 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:22,468 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:50:22,608 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:22,617 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:24,853 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:50:25,350 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:25,359 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:25,536 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:27,377 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:27,386 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:29,940 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:29,952 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:30,818 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:31,968 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:31,979 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:32,856 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:50:33,521 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:50:33,522 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:50:34,557 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:34,566 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:36,586 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:36,598 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:36,674 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:37,468 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:50:39,689 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:39,700 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:40,858 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:50:41,716 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:41,725 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:42,199 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:44,203 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:44,212 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:44,699 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:50:46,231 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:46,242 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:47,478 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:48,860 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:50:49,162 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:49,173 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:51,191 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:51,200 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:52,478 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:52,483 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:50:52,483 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:50:52,552 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:50:54,477 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:50:54,500 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:54,521 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:56,862 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:50:57,088 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:57,099 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:57,741 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:50:59,116 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:50:59,127 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:01,704 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:01,714 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:02,741 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:03,399 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:51:03,399 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:51:03,732 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:03,743 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:06,312 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:06,322 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:06,866 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:51:08,035 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:08,338 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:08,346 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:09,122 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:51:11,011 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:11,026 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:13,052 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:13,060 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:13,334 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:14,700 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:51:14,868 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:51:15,692 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:15,702 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:17,718 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:17,726 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:18,399 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:51:18,399 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:51:18,542 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:20,721 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:20,731 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:22,756 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:22,766 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:22,870 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:51:23,689 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:24,192 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:51:25,152 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:25,161 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:27,187 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:27,198 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:28,813 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:30,393 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:51:30,415 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:30,425 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:30,872 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:51:32,853 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:32,863 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:34,009 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:34,163 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:51:34,163 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:51:34,881 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:34,894 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:37,867 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:37,876 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:38,875 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:51:39,333 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:39,892 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:39,905 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:42,495 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:42,504 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:42,510 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:51:44,527 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:44,537 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:44,567 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:44,702 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:51:46,877 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:51:47,506 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:47,515 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:49,535 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:49,545 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:49,777 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:50,986 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:51:50,986 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:51:52,421 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:52,434 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:53,308 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:51:54,451 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:54,460 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:55,153 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:51:56,880 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:51:57,152 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:57,162 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:59,186 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:51:59,195 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:00,486 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:02,080 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:02,090 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:04,107 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:04,116 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:04,882 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:52:06,432 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:07,055 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:52:07,073 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:07,083 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:07,410 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:52:07,410 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:52:08,308 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:52:09,893 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:09,903 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:11,651 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:11,921 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:11,930 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:12,885 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:52:14,525 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:14,537 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:14,703 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:52:16,555 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:16,564 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:16,705 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:19,424 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:19,433 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:20,887 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:52:21,452 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:21,464 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:22,016 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:23,406 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:52:23,406 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:52:23,945 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:23,954 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:24,178 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:52:25,975 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:25,989 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:27,091 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:28,820 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:28,829 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:28,890 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:52:30,846 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:30,856 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:32,377 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:33,244 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:33,254 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:35,275 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:35,286 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:36,892 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:52:37,678 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:37,957 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:37,967 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:39,166 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:52:39,167 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:52:39,993 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:40,003 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:40,753 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:52:42,836 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:43,241 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:52:43,273 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:43,289 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:44,705 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:52:44,895 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:52:45,867 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:45,876 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:47,903 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:47,915 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:48,706 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:50,740 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:50,754 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:51,601 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:52:51,602 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:52:52,773 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:52,782 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:54,325 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:52:54,899 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:52:55,458 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:55,467 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:55,865 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:52:57,484 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:57,494 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:52:59,456 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:00,146 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:00,156 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:02,174 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:02,186 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:02,901 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:53:04,702 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:04,761 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:04,771 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:06,602 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:53:06,602 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:53:06,791 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:06,801 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:09,619 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:09,646 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:10,016 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:10,903 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:53:11,664 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:11,676 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:13,395 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:53:14,213 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:14,223 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:14,707 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:53:15,708 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:16,248 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:16,260 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:17,413 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 12:53:17,414 DEBUG SenderThread:1368934 [sender.py:send():382] send: history +2024-03-27 12:53:17,414 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: summary_record +2024-03-27 12:53:17,415 INFO SenderThread:1368934 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 12:53:17,905 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:53:17,906 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json +2024-03-27 12:53:18,906 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:53:19,587 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:53:19,607 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:19,618 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:21,415 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:22,032 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:22,045 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:22,550 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:53:22,550 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:53:24,065 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:24,081 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:26,730 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:26,750 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:26,758 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:26,908 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:53:28,780 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:28,790 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:30,803 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:53:31,369 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:31,378 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:32,252 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:33,403 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:33,413 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:34,910 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:53:35,434 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:35,443 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:37,821 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:37,830 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:38,069 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:39,129 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:53:39,129 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:53:39,862 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:39,872 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:41,661 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:53:42,756 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:42,765 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:43,366 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:44,708 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:53:44,788 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:44,797 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:44,913 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:53:47,282 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:47,293 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:48,861 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:49,311 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:49,322 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:52,168 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:52,178 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:52,915 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:53:54,690 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:53:55,531 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:53:55,549 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:55,560 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:55,829 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:53:55,830 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:53:56,662 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:53:58,285 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:53:58,298 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:00,095 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:00,317 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:00,334 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:00,918 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:54:02,895 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:02,904 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:04,926 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:04,941 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:05,096 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:07,818 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:07,826 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:08,920 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:54:09,848 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:09,860 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:10,101 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:54:10,102 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:54:10,302 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:12,434 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:12,443 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:12,610 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:54:14,464 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:14,474 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:14,710 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:54:15,711 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:16,923 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:54:17,342 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:17,352 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:19,370 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:19,381 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:20,791 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:21,908 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:21,918 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:23,945 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:23,957 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:24,929 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:54:26,168 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:26,601 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:26,610 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:26,825 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:54:26,825 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:54:28,514 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:54:28,629 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:28,637 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:31,346 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:31,784 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:54:31,807 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:31,818 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:34,527 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:34,536 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:34,932 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:54:36,465 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:36,557 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:36,565 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:39,320 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:39,336 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:40,117 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:54:40,118 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:54:41,353 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:41,361 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:41,722 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:42,935 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:54:44,086 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:44,096 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:44,711 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:54:45,865 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:54:46,121 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:46,131 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:46,935 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:48,833 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:48,848 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:50,866 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:50,875 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:50,937 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:54:52,079 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:53,398 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:53,407 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:55,118 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:54:55,118 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:54:55,427 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:55,443 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:57,279 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:54:58,180 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:58,189 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:54:58,940 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:55:00,206 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:00,217 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:02,514 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:55:02,750 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:02,758 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:02,779 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:55:04,778 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:04,791 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:06,942 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:55:08,194 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:55:08,214 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:08,224 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:08,719 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:55:10,639 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:10,648 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:10,856 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:55:10,856 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:55:12,672 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:12,682 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:14,056 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:55:14,713 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:55:14,944 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:55:15,536 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:15,548 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:17,567 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:17,576 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:18,492 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 12:55:19,759 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:55:20,061 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:20,074 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:22,094 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:22,107 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:24,714 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:24,725 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:24,947 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:55:25,459 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:55:26,742 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:26,752 ERROR gpu :1368934 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 12:55:27,601 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 12:55:27,602 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: stop_status +2024-03-27 12:55:28,154 DEBUG SenderThread:1368934 [sender.py:send():382] send: exit +2024-03-27 12:55:28,155 INFO SenderThread:1368934 [sender.py:send_exit():589] handling exit code: 255 +2024-03-27 12:55:28,155 INFO SenderThread:1368934 [sender.py:send_exit():591] handling runtime: 1183 +2024-03-27 12:55:28,155 INFO SenderThread:1368934 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 12:55:28,155 INFO SenderThread:1368934 [sender.py:send_exit():597] send defer +2024-03-27 12:55:28,155 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: defer +2024-03-27 12:55:28,155 INFO HandlerThread:1368934 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-27 12:55:28,156 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: defer +2024-03-27 12:55:28,156 INFO SenderThread:1368934 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-27 12:55:28,156 INFO SenderThread:1368934 [sender.py:transition_state():617] send defer: 1 +2024-03-27 12:55:28,156 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: defer +2024-03-27 12:55:28,156 INFO HandlerThread:1368934 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-27 12:55:28,156 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: defer +2024-03-27 12:55:28,156 INFO SenderThread:1368934 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-27 12:55:28,156 INFO SenderThread:1368934 [sender.py:transition_state():617] send defer: 2 +2024-03-27 12:55:28,156 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: defer +2024-03-27 12:55:28,156 INFO HandlerThread:1368934 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-27 12:55:28,156 INFO HandlerThread:1368934 [system_monitor.py:finish():203] Stopping system monitor +2024-03-27 12:55:28,157 DEBUG SystemMonitor:1368934 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-27 12:55:28,157 DEBUG SystemMonitor:1368934 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-27 12:55:28,159 INFO HandlerThread:1368934 [interfaces.py:finish():202] Joined cpu monitor +2024-03-27 12:55:28,160 INFO HandlerThread:1368934 [interfaces.py:finish():202] Joined disk monitor +2024-03-27 12:55:28,949 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:55:28,949 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json +2024-03-27 12:55:29,280 ERROR gpu :1368934 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 12:55:29,280 INFO HandlerThread:1368934 [interfaces.py:finish():202] Joined gpu monitor +2024-03-27 12:55:29,280 INFO HandlerThread:1368934 [interfaces.py:finish():202] Joined memory monitor +2024-03-27 12:55:29,280 INFO HandlerThread:1368934 [interfaces.py:finish():202] Joined network monitor +2024-03-27 12:55:29,281 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-27 12:55:29,281 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: defer +2024-03-27 12:55:29,281 INFO SenderThread:1368934 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-27 12:55:29,281 INFO SenderThread:1368934 [sender.py:transition_state():617] send defer: 3 +2024-03-27 12:55:29,281 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: defer +2024-03-27 12:55:29,281 INFO HandlerThread:1368934 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-27 12:55:29,282 DEBUG SenderThread:1368934 [sender.py:send():382] send: stats +2024-03-27 12:55:29,283 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: poll_exit +2024-03-27 12:55:29,283 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: defer +2024-03-27 12:55:29,283 INFO SenderThread:1368934 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-27 12:55:29,283 INFO SenderThread:1368934 [sender.py:transition_state():617] send defer: 4 +2024-03-27 12:55:29,283 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: defer +2024-03-27 12:55:29,283 INFO HandlerThread:1368934 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-27 12:55:29,284 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: defer +2024-03-27 12:55:29,284 INFO SenderThread:1368934 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-27 12:55:29,284 INFO SenderThread:1368934 [sender.py:transition_state():617] send defer: 5 +2024-03-27 12:55:29,284 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: defer +2024-03-27 12:55:29,284 INFO HandlerThread:1368934 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-27 12:55:29,284 DEBUG SenderThread:1368934 [sender.py:send():382] send: summary +2024-03-27 12:55:29,284 INFO SenderThread:1368934 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 12:55:29,285 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: defer +2024-03-27 12:55:29,285 INFO SenderThread:1368934 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-27 12:55:29,285 INFO SenderThread:1368934 [sender.py:transition_state():617] send defer: 6 +2024-03-27 12:55:29,285 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: defer +2024-03-27 12:55:29,285 INFO HandlerThread:1368934 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-27 12:55:29,285 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: defer +2024-03-27 12:55:29,285 INFO SenderThread:1368934 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-27 12:55:29,285 INFO SenderThread:1368934 [sender.py:transition_state():617] send defer: 7 +2024-03-27 12:55:29,285 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 12:55:29,285 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: defer +2024-03-27 12:55:29,285 INFO HandlerThread:1368934 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-27 12:55:29,285 DEBUG SenderThread:1368934 [sender.py:send_request():409] send_request: defer +2024-03-27 12:55:29,286 INFO SenderThread:1368934 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-27 12:55:29,949 INFO Thread-13 :1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json +2024-03-27 12:55:30,155 DEBUG HandlerThread:1368934 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-27 12:55:30,306 WARNING StreamThr :1368934 [internal.py:is_dead():414] Internal process exiting, parent pid 1366875 disappeared +2024-03-27 12:55:30,306 ERROR StreamThr :1368934 [internal.py:wandb_internal():152] Internal process shutdown. +2024-03-27 12:55:31,155 INFO HandlerThread:1368934 [handler.py:finish():869] shutting down handler +2024-03-27 12:55:31,155 INFO WriterThread:1368934 [datastore.py:close():296] close: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/run-5vwxt2ut.wandb +2024-03-27 12:55:32,032 INFO SenderThread:1368934 [sender.py:transition_state():617] send defer: 8 +2024-03-27 12:55:32,032 INFO SenderThread:1368934 [sender.py:finish():1572] shutting down sender +2024-03-27 12:55:32,032 INFO SenderThread:1368934 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [dir_watcher.py:finish():388] scan: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/requirements.txt requirements.txt +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-metadata.json wandb-metadata.json +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json wandb-summary.json +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log output.log +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [dir_watcher.py:finish():402] scan save: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/config.yaml config.yaml +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [file_pusher.py:finish():175] shutting down file pusher +2024-03-27 12:55:32,951 INFO SenderThread:1368934 [file_pusher.py:join():181] waiting for file pusher +2024-03-27 12:55:33,309 INFO wandb-upload_0:1368934 [upload_job.py:push():131] Uploaded file /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/requirements.txt +2024-03-27 12:55:33,332 INFO wandb-upload_1:1368934 [upload_job.py:push():131] Uploaded file /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/wandb-summary.json +2024-03-27 12:55:33,338 INFO wandb-upload_2:1368934 [upload_job.py:push():131] Uploaded file /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/output.log +2024-03-27 12:55:33,371 INFO wandb-upload_3:1368934 [upload_job.py:push():131] Uploaded file /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_123544-5vwxt2ut/files/config.yaml +2024-03-27 12:55:33,954 INFO SenderThread:1368934 [file_stream.py:finish():595] file stream finish called +2024-03-27 12:55:34,493 INFO SenderThread:1368934 [file_stream.py:finish():599] file stream finish is done