diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,20299 +1,9954 @@ -2024-03-27 14:10:31,572 INFO StreamThr :1483352 [internal.py:wandb_internal():86] W&B internal server running at pid: 1483352, started at: 2024-03-27 14:10:31.571470 -2024-03-27 14:10:31,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status -2024-03-27 14:10:31,576 INFO WriterThread:1483352 [datastore.py:open_for_write():87] open: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/run-aoxf8fxn.wandb -2024-03-27 14:10:31,577 DEBUG SenderThread:1483352 [sender.py:send():382] send: header -2024-03-27 14:10:31,632 DEBUG SenderThread:1483352 [sender.py:send():382] send: run -2024-03-27 14:10:31,970 INFO SenderThread:1483352 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files -2024-03-27 14:10:31,970 INFO SenderThread:1483352 [sender.py:_start_run_threads():1136] run started: aoxf8fxn with start time 1711545031.570969 -2024-03-27 14:10:31,977 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: check_version -2024-03-27 14:10:31,977 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: check_version -2024-03-27 14:10:32,006 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: run_start -2024-03-27 14:10:32,448 DEBUG HandlerThread:1483352 [system_info.py:__init__():27] System info init -2024-03-27 14:10:32,448 DEBUG HandlerThread:1483352 [system_info.py:__init__():42] System info init done -2024-03-27 14:10:32,448 INFO HandlerThread:1483352 [system_monitor.py:start():194] Starting system monitor -2024-03-27 14:10:32,448 INFO SystemMonitor:1483352 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-03-27 14:10:32,449 INFO HandlerThread:1483352 [system_monitor.py:probe():214] Collecting system info -2024-03-27 14:10:32,450 INFO SystemMonitor:1483352 [interfaces.py:start():190] Started cpu monitoring -2024-03-27 14:10:32,450 INFO SystemMonitor:1483352 [interfaces.py:start():190] Started disk monitoring -2024-03-27 14:10:32,452 INFO SystemMonitor:1483352 [interfaces.py:start():190] Started gpu monitoring -2024-03-27 14:10:32,453 INFO SystemMonitor:1483352 [interfaces.py:start():190] Started memory monitoring -2024-03-27 14:10:32,455 INFO SystemMonitor:1483352 [interfaces.py:start():190] Started network monitoring -2024-03-27 14:10:32,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:32,506 DEBUG HandlerThread:1483352 [system_info.py:probe():151] Probing system -2024-03-27 14:10:32,511 DEBUG HandlerThread:1483352 [system_info.py:_probe_git():136] Probing git -2024-03-27 14:10:32,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:32,521 DEBUG HandlerThread:1483352 [system_info.py:_probe_git():144] Probing git done -2024-03-27 14:10:32,521 DEBUG HandlerThread:1483352 [system_info.py:probe():199] Probing system done -2024-03-27 14:10:32,521 DEBUG HandlerThread:1483352 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-03-27T13:10:32.506516', 'startedAt': '2024-03-27T13:10:31.567196', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=distil-whisper/distil-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_1', '--dataset_config_name=hi', '--language=hindi', '--train_split_name=train+validation', '--eval_split_name=test', '--max_steps=5000', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=32', '--logging_steps=25', '--learning_rate=1e-4', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--generation_max_length=225', '--preprocessing_num_workers=1', '--dataloader_num_workers=4', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--gradient_checkpointing', '--fp16', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-large-v3-hi-ft', 'commit': '40c686df113c0e98e7363c1bd523f58d11848fc0'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/distil-large-v3-hi-ft', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2591.1360703124997, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2947.674, 'min': 1500.0, 'max': 2250.0}, {'current': 2021.008, 'min': 1500.0, 'max': 2250.0}, {'current': 2933.218, 'min': 1500.0, 'max': 2250.0}, {'current': 2938.11, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.021, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.107, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.106, 'min': 1500.0, 'max': 2250.0}, {'current': 2935.616, 'min': 1500.0, 'max': 2250.0}, {'current': 2936.719, 'min': 1500.0, 'max': 2250.0}, {'current': 2950.399, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.429, 'min': 1500.0, 'max': 2250.0}, {'current': 2935.347, 'min': 1500.0, 'max': 2250.0}, {'current': 2938.48, 'min': 1500.0, 'max': 2250.0}, {'current': 2947.593, 'min': 1500.0, 'max': 2250.0}, {'current': 2944.163, 'min': 1500.0, 'max': 2250.0}, {'current': 1688.785, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.151, 'min': 1500.0, 'max': 2250.0}, {'current': 2952.753, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.208, 'min': 1500.0, 'max': 2250.0}, {'current': 1868.923, 'min': 1500.0, 'max': 2250.0}, {'current': 2959.785, 'min': 1500.0, 'max': 2250.0}, {'current': 2937.134, 'min': 1500.0, 'max': 2250.0}, {'current': 2936.912, 'min': 1500.0, 'max': 2250.0}, {'current': 1982.456, 'min': 1500.0, 'max': 2250.0}, {'current': 2945.492, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.291, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.233, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.064, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.272, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.118, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.057, 'min': 1500.0, 'max': 2250.0}, {'current': 2941.903, 'min': 1500.0, 'max': 2250.0}, {'current': 2950.964, 'min': 1500.0, 'max': 2250.0}, {'current': 2952.685, 'min': 1500.0, 'max': 2250.0}, {'current': 2944.184, 'min': 1500.0, 'max': 2250.0}, {'current': 2945.032, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.66, 'min': 1500.0, 'max': 2250.0}, {'current': 2941.93, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.832, 'min': 1500.0, 'max': 2250.0}, {'current': 2944.647, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.166, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.746, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.47, 'min': 1500.0, 'max': 2250.0}, {'current': 2944.65, 'min': 1500.0, 'max': 2250.0}, {'current': 2937.31, 'min': 1500.0, 'max': 2250.0}, {'current': 2948.814, 'min': 1500.0, 'max': 2250.0}, {'current': 2939.028, 'min': 1500.0, 'max': 2250.0}, {'current': 2938.291, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.541, 'min': 1500.0, 'max': 2250.0}, {'current': 2948.065, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.119, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.127, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.091, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.402, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.105, 'min': 1500.0, 'max': 2250.0}, {'current': 2949.13, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.763, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.78, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.786, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.657, 'min': 1500.0, 'max': 2250.0}, {'current': 1681.397, 'min': 1500.0, 'max': 2250.0}, {'current': 2937.278, 'min': 1500.0, 'max': 2250.0}, {'current': 2951.047, 'min': 1500.0, 'max': 2250.0}, {'current': 1680.886, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.17, 'min': 1500.0, 'max': 2250.0}, {'current': 1754.766, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.394, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.181, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.186, 'min': 1500.0, 'max': 2250.0}, {'current': 2945.466, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.336, 'min': 1500.0, 'max': 2250.0}, {'current': 2943.198, 'min': 1500.0, 'max': 2250.0}, {'current': 2926.611, 'min': 1500.0, 'max': 2250.0}, {'current': 2939.83, 'min': 1500.0, 'max': 2250.0}, {'current': 2939.84, 'min': 1500.0, 'max': 2250.0}, {'current': 2948.356, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.765, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.047, 'min': 1500.0, 'max': 2250.0}, {'current': 2469.696, 'min': 1500.0, 'max': 2250.0}, {'current': 1593.287, 'min': 1500.0, 'max': 2250.0}, {'current': 2952.788, 'min': 1500.0, 'max': 2250.0}, {'current': 2941.794, 'min': 1500.0, 'max': 2250.0}, {'current': 2949.289, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.056, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.118, 'min': 1500.0, 'max': 2250.0}, {'current': 2944.309, 'min': 1500.0, 'max': 2250.0}, {'current': 2952.465, 'min': 1500.0, 'max': 2250.0}, {'current': 1682.593, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.404, 'min': 1500.0, 'max': 2250.0}, {'current': 2960.474, 'min': 1500.0, 'max': 2250.0}, {'current': 2948.539, 'min': 1500.0, 'max': 2250.0}, {'current': 2953.446, 'min': 1500.0, 'max': 2250.0}, {'current': 2968.584, 'min': 1500.0, 'max': 2250.0}, {'current': 2968.447, 'min': 1500.0, 'max': 2250.0}, {'current': 2954.762, 'min': 1500.0, 'max': 2250.0}, {'current': 2936.182, 'min': 1500.0, 'max': 2250.0}, {'current': 2944.46, 'min': 1500.0, 'max': 2250.0}, {'current': 2944.181, 'min': 1500.0, 'max': 2250.0}, {'current': 2958.541, 'min': 1500.0, 'max': 2250.0}, {'current': 2948.457, 'min': 1500.0, 'max': 2250.0}, {'current': 2953.617, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.23, 'min': 1500.0, 'max': 2250.0}, {'current': 2948.216, 'min': 1500.0, 'max': 2250.0}, {'current': 2941.896, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.828, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.837, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.776, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.719, 'min': 1500.0, 'max': 2250.0}, {'current': 2938.813, 'min': 1500.0, 'max': 2250.0}, {'current': 2938.962, 'min': 1500.0, 'max': 2250.0}, {'current': 2939.034, 'min': 1500.0, 'max': 2250.0}, {'current': 2925.934, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.86, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.866, 'min': 1500.0, 'max': 2250.0}, {'current': 2938.896, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.875, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.101, 'min': 1500.0, 'max': 2250.0}, {'current': 2949.598, 'min': 1500.0, 'max': 2250.0}, {'current': 2935.583, 'min': 1500.0, 'max': 2250.0}, {'current': 2942.305, 'min': 1500.0, 'max': 2250.0}, {'current': 2853.409, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.957, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.961, 'min': 1500.0, 'max': 2250.0}, {'current': 2949.248, 'min': 1500.0, 'max': 2250.0}, {'current': 1680.132, 'min': 1500.0, 'max': 2250.0}, {'current': 2673.469, 'min': 1500.0, 'max': 2250.0}, {'current': 2940.203, 'min': 1500.0, 'max': 2250.0}, {'current': 1681.826, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1516.8664588928223}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} -2024-03-27 14:10:32,522 INFO HandlerThread:1483352 [system_monitor.py:probe():224] Finished collecting system info -2024-03-27 14:10:32,522 INFO HandlerThread:1483352 [system_monitor.py:probe():227] Publishing system info -2024-03-27 14:10:32,523 INFO HandlerThread:1483352 [system_monitor.py:probe():229] Finished publishing system info -2024-03-27 14:10:32,526 DEBUG SenderThread:1483352 [sender.py:send():382] send: files -2024-03-27 14:10:32,526 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now -2024-03-27 14:10:32,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: python_packages -2024-03-27 14:10:32,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:10:32,531 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: python_packages -2024-03-27 14:10:32,532 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:10:32,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:10:32,701 DEBUG SenderThread:1483352 [sender.py:send():382] send: telemetry -2024-03-27 14:10:32,701 DEBUG SenderThread:1483352 [sender.py:send():382] send: config -2024-03-27 14:10:32,702 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 14:10:32,702 DEBUG SenderThread:1483352 [sender.py:send():382] send: telemetry -2024-03-27 14:10:32,702 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 14:10:32,702 WARNING SenderThread:1483352 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) -2024-03-27 14:10:32,702 DEBUG SenderThread:1483352 [sender.py:send():382] send: telemetry -2024-03-27 14:10:32,972 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/requirements.txt -2024-03-27 14:10:32,972 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-metadata.json -2024-03-27 14:10:32,972 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:10:33,043 INFO wandb-upload_0:1483352 [upload_job.py:push():131] Uploaded file /tmp/tmpcn2z1k5pwandb/vfsn8kdj-wandb-metadata.json -2024-03-27 14:10:34,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:34,971 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:10:35,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:36,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:37,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:37,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:39,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:39,986 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:41,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:42,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:42,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:44,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:44,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:46,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:46,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:46,976 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:10:47,122 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:47,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:10:47,532 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:10:47,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:10:49,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:49,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:51,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:51,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:52,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:54,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:54,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:54,979 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:10:56,370 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:56,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:57,789 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:10:59,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:10:59,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:00,981 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:01,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:01,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:02,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:11:02,532 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:11:02,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:11:03,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:03,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:03,982 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/config.yaml -2024-03-27 14:11:04,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:04,982 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:06,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:06,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:06,983 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:09,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:09,773 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:11:09,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:09,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:12,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:12,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:12,985 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:14,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:14,786 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:15,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:17,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:11:17,532 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:11:17,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:11:17,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:17,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:18,987 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:19,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:19,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:20,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:22,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:22,634 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:24,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:24,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:24,989 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:26,626 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:27,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:27,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:29,584 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:29,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:30,990 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:31,648 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:32,455 DEBUG SystemMonitor:1483352 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-03-27 14:11:32,458 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:11:32,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:11:32,532 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:11:32,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:11:32,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:32,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:34,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:34,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:36,665 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:36,992 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:37,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:37,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:39,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:39,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:41,666 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:42,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:42,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:42,994 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:44,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:44,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:46,713 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:47,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:11:47,532 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:11:47,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:11:48,234 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:11:48,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:48,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:48,996 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:51,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:51,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:51,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:53,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:53,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:54,998 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:11:56,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:56,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:56,767 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:11:58,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:11:58,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:01,000 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:01,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:01,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:01,809 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:02,460 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:12:02,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:12:02,532 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:12:02,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:12:03,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:03,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:06,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:06,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:06,846 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:07,003 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:08,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:08,197 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:11,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:11,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:11,847 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:13,005 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:13,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:13,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:16,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:16,129 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:16,875 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:17,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:12:17,532 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:12:17,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:12:18,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:18,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:21,007 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:21,024 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:21,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:22,749 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:23,054 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:23,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:25,009 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:26,504 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:12:26,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:27,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:28,007 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:29,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:29,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:31,011 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:32,462 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:12:32,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:12:32,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:12:32,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:12:32,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:32,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:33,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:34,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:34,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:37,013 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:37,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:37,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:39,088 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:39,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:39,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:42,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:42,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:43,015 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:44,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:44,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:44,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:47,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:12:47,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:12:47,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:12:47,676 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:47,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:49,017 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:49,162 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:49,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:49,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:52,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:52,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:54,164 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:54,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:54,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:55,019 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:12:57,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:57,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:59,215 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:12:59,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:12:59,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:01,021 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:02,463 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:13:02,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:13:02,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:13:02,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:13:02,773 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:02,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:04,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:06,402 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:13:06,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:06,430 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:07,023 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:09,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:09,591 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:10,288 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:11,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:11,647 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:12,425 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:13:12,427 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 14:13:12,428 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 14:13:12,428 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 14:13:12,428 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 14:13:12,428 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:13:12,428 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:13:12,429 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:13:13,025 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:13,025 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:13:14,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:14,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:15,026 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:15,430 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:16,557 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:16,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:17,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:13:17,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:13:17,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:13:18,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:18,615 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:19,027 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:21,399 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:21,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:21,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:23,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:23,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:25,029 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:26,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:26,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:26,648 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:28,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:28,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:31,031 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:31,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:31,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:31,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:32,465 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:13:32,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:13:32,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:13:32,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:13:33,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:33,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:36,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:36,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:36,751 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:37,033 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:38,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:38,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:41,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:41,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:41,755 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:42,035 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/config.yaml -2024-03-27 14:13:43,035 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:45,168 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:13:45,189 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:45,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:47,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:13:47,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:13:47,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:13:48,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:48,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:50,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:50,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:50,883 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:51,038 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:53,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:53,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:55,039 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:13:55,100 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:55,120 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:55,910 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:13:58,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:13:58,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:00,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:00,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:00,911 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:02,468 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:14:02,532 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:14:02,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:14:02,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:14:03,042 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:03,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:03,128 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:05,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:05,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:06,750 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:08,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:08,137 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:09,044 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:10,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:10,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:12,039 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:13,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:13,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:15,046 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:15,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:15,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:17,091 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:17,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:14:17,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:14:17,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:14:18,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:18,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:20,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:20,110 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:21,048 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:22,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:23,630 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:14:23,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:23,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:26,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:26,669 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:27,051 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:27,210 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:28,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:28,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:31,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:31,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:32,277 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:32,469 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:14:32,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:14:32,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:14:32,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:14:33,053 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:33,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:33,905 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:36,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:36,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:37,336 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:38,934 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:38,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:39,055 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:41,786 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:41,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:42,337 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:43,848 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:43,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:45,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:45,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:46,057 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:47,385 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:47,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:14:47,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:14:47,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:14:48,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:48,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:50,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:50,894 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:52,059 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:14:52,441 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:52,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:52,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:55,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:55,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:57,490 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:14:58,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:58,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:14:58,061 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:01,680 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:15:02,471 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:15:02,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:02,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:02,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:15:02,533 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:15:02,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:15:02,752 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:04,063 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:04,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:04,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:07,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:07,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:08,605 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:09,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:09,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:10,065 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:12,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:12,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:13,606 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:14,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:14,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:16,067 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:17,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:15:17,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:15:17,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:15:17,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:17,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:18,673 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:19,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:19,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:22,070 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:22,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:22,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:23,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:24,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:24,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:27,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:27,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:28,072 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:28,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:29,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:29,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:32,473 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:15:32,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:15:32,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:15:32,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:15:32,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:32,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:34,074 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:34,657 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:34,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:34,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:37,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:37,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:39,896 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:40,076 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:41,331 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:15:41,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:41,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:43,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:43,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:44,029 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:15:44,030 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:15:44,031 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:15:44,031 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:15:44,077 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:15:45,032 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:45,078 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:46,078 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:46,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:46,189 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:47,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:15:47,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:15:47,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:15:48,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:48,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:51,026 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:51,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:51,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:52,080 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:53,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:53,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:56,027 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:15:56,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:56,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:58,082 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:15:58,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:15:58,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:00,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:00,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:01,083 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:02,475 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:16:02,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:16:02,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:16:02,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:16:03,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:03,063 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:04,084 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:05,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:05,094 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:06,754 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:08,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:08,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:10,086 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:10,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:10,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:12,200 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:13,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:13,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:15,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:15,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:16,088 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:17,260 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:17,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:16:17,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:16:17,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:16:19,194 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:16:19,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:19,230 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:22,090 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:22,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:22,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:22,322 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:24,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:24,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:27,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:27,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:27,390 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:28,092 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:29,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:29,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:32,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:32,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:32,391 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:32,476 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:16:32,533 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:16:32,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:16:32,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:16:34,094 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:34,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:34,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:36,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:36,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:37,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:39,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:39,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:40,096 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:41,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:41,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:43,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:43,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:43,509 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:46,098 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:46,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:46,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:47,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:16:47,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:16:47,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:16:48,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:48,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:48,747 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:51,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:51,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:52,100 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:53,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:53,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:54,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:16:56,779 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:16:57,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:57,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:58,102 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:16:59,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:16:59,692 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:00,188 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:01,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:01,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:02,103 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:02,478 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:17:02,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:17:02,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:17:02,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:17:04,104 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:04,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:04,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:05,676 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:06,105 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:06,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:06,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:09,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:09,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:10,106 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:11,408 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:11,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:11,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:14,108 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:14,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:14,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:16,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:16,773 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:16,883 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:17,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:17:17,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:17:17,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:17:19,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:19,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:20,110 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:21,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:21,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:22,637 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:24,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:24,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:26,112 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:26,933 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:26,948 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:27,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:29,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:29,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:31,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:31,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:32,114 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:32,479 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:17:32,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:17:32,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:17:32,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:17:33,678 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:35,421 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:17:35,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:35,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:38,116 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:38,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:38,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:38,829 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:40,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:40,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:43,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:43,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:43,904 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:44,118 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:45,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:45,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:47,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:17:47,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:17:47,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:17:48,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:48,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:48,953 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:50,120 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:50,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:50,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:53,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:53,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:53,954 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:17:55,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:55,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:56,122 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:17:57,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:57,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:17:59,149 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:00,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:00,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:01,290 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:18:01,292 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:18:01,292 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:18:01,293 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:18:02,124 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:18:02,125 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:02,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:02,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:02,481 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:18:02,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:18:02,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:18:02,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:18:03,125 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:04,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:05,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:05,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:07,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:07,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:10,128 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:10,289 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:10,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:10,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:14,157 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:18:14,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:14,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:15,359 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:16,130 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:17,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:17,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:17,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:18:17,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:18:17,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:18:19,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:19,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:20,432 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:22,132 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:22,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:22,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:24,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:24,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:25,432 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:27,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:27,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:28,134 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:29,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:29,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:30,502 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:32,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:32,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:32,483 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:18:32,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:18:32,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:18:32,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:18:34,136 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:34,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:34,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:35,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:36,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:36,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:39,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:39,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:40,138 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:41,332 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:41,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:41,635 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:44,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:44,224 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:46,140 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:46,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:46,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:46,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:47,534 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:18:47,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:18:47,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:18:49,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:49,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:51,751 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:52,142 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:18:52,919 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:18:52,953 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:52,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:55,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:55,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:56,836 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:18:57,827 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:57,848 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:18:58,145 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:00,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:00,934 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:01,837 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:02,484 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:19:02,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:19:02,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:19:02,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:19:02,960 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:02,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:04,147 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:04,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:05,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:07,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:07,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:07,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:09,965 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:09,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:10,149 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:12,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:12,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:12,957 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:14,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:14,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:16,151 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:17,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:19:17,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:19:17,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:19:17,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:17,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:18,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:19,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:19,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:22,153 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:22,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:22,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:24,088 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:24,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:24,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:27,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:27,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:28,155 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:29,152 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:31,282 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:19:31,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:31,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:32,486 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:19:32,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:19:32,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:19:32,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:19:34,157 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:34,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:34,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:34,755 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:36,428 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:36,443 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:39,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:39,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:40,159 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:40,282 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:41,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:41,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:44,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:44,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:45,345 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:46,161 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:46,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:46,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:47,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:19:47,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:19:47,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:19:49,428 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:49,458 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:50,414 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:51,505 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:51,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:52,163 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:54,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:54,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:55,415 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:19:56,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:56,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:58,165 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:19:59,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:19:59,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:00,479 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:01,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:01,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:02,487 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:20:02,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:20:02,535 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:20:02,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:20:04,167 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:04,588 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:04,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:05,657 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:06,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:06,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:10,126 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:20:10,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:10,169 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:10,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:11,619 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:13,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:13,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:15,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:15,128 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:16,171 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:16,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:17,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:20:17,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:20:17,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:20:18,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:18,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:20,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:20,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:21,752 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:22,174 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:23,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:23,258 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:25,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:25,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:26,753 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:28,175 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:28,417 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:28,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:30,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:30,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:31,831 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:32,489 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:20:32,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:20:32,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:20:32,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:20:32,957 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:20:32,959 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:20:32,959 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:20:32,959 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:20:33,177 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:20:33,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:33,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:34,178 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:35,178 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:35,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:35,445 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:36,960 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:38,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:38,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:40,179 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:40,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:40,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:42,947 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:43,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:43,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:45,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:45,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:46,181 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:47,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:20:47,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:20:47,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:20:48,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:48,934 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:20:48,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:48,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:51,684 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:51,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:52,184 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:53,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:53,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:54,082 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:20:56,789 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:56,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:58,185 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:20:58,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:58,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:20:59,134 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:01,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:01,816 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:02,491 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:21:02,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:21:02,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:21:02,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:21:03,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:03,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:04,187 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:04,206 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:06,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:06,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:08,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:08,903 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:09,207 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:10,189 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:11,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:11,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:13,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:13,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:14,255 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:16,192 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:16,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:16,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:17,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:21:17,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:21:17,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:21:18,882 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:18,893 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:19,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:21,882 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:21,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:22,194 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:23,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:23,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:25,388 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:28,197 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:28,420 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:21:28,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:28,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:30,446 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:30,717 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:30,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:32,493 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:21:32,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:21:32,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:21:32,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:21:33,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:33,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:35,523 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:35,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:35,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:36,200 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:38,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:38,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:40,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:40,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:40,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:42,203 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:43,585 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:43,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:45,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:45,625 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:45,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:47,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:21:47,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:21:47,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:21:48,205 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:48,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:48,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:50,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:50,588 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:50,657 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:53,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:53,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:54,207 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:21:55,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:55,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:55,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:21:58,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:21:58,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:00,209 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:00,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:00,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:00,788 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:02,494 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:22:02,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:22:02,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:22:02,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:22:03,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:03,496 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:05,844 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:06,211 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:07,214 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:22:07,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:07,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:10,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:10,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:10,925 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:12,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:12,213 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:12,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:15,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:15,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:15,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:17,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:17,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:17,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:22:17,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:22:17,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:22:18,215 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:20,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:20,230 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:21,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:22,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:22,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:24,217 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:25,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:25,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:27,056 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:27,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:27,179 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:30,148 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:30,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:30,219 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:32,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:32,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:32,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:32,495 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:22:32,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:22:32,536 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:22:32,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:22:35,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:35,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:36,222 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:37,211 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:37,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:37,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:40,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:40,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:42,224 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:42,273 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:42,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:42,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:45,920 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:22:45,972 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:45,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:47,354 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:47,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:22:47,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:22:47,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:22:48,226 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:48,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:48,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:50,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:50,860 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:52,414 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:53,801 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:53,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:54,229 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:22:55,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:55,871 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:57,416 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:22:58,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:22:58,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:00,231 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:00,735 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:00,755 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:02,461 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:02,497 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:23:02,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:23:02,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:23:02,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:23:03,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:03,848 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:04,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:23:04,598 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:23:04,599 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:23:04,599 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:23:05,232 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:23:05,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:05,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:06,234 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:07,234 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:07,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:08,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:08,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:10,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:10,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:12,236 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:13,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:13,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:13,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:15,786 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:15,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:17,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:23:17,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:23:17,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:23:18,239 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:18,755 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:18,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:18,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:20,239 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:20,975 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:20,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:23,914 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:24,241 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:24,458 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:23:24,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:24,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:26,242 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:27,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:27,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:28,242 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:29,393 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:29,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:29,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:32,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:32,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:32,498 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:23:32,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:23:32,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:23:32,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:23:34,245 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:34,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:34,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:34,870 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:37,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:37,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:39,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:39,364 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:40,247 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:40,677 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:42,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:42,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:44,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:44,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:45,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:46,249 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:47,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:47,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:47,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:23:47,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:23:47,540 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:23:49,417 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:49,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:50,795 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:52,251 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:52,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:52,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:54,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:54,633 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:55,795 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:23:57,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:57,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:58,253 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:23:59,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:23:59,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:00,856 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:02,500 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:24:02,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:24:02,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:24:02,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:24:04,054 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:24:04,255 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:04,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:04,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:05,923 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:06,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:06,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:08,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:08,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:10,257 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:11,082 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:11,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:11,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:13,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:13,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:16,153 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:16,260 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:16,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:16,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:17,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:24:17,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:24:17,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:24:18,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:18,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:21,216 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:21,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:21,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:22,262 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:23,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:23,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:26,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:26,217 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:26,225 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:28,264 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:28,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:28,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:31,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:31,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:31,273 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:32,501 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:24:32,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:24:32,537 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:24:32,540 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:24:33,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:33,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:34,266 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:36,158 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:36,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:36,319 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:38,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:38,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:40,268 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:41,372 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:41,645 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:24:41,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:41,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:44,414 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:44,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:46,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:46,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:46,928 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:47,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:24:47,538 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:24:47,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:24:48,270 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:48,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:48,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:51,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:51,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:51,990 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:53,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:53,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:54,272 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:24:55,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:55,391 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:56,990 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:24:58,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:24:58,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:00,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:00,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:00,274 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:02,050 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:02,503 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:25:02,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:25:02,538 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:25:02,540 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:25:03,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:03,207 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:05,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:05,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:06,275 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:07,110 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:08,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:08,222 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:10,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:10,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:12,175 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:12,277 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:13,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:13,112 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:15,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:15,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:17,228 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:17,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:25:17,538 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:25:17,540 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:25:18,279 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:18,905 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:25:19,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:19,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:21,804 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:21,817 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:22,273 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:22,344 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:25:22,345 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:25:22,346 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:25:22,346 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:25:23,280 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:25:23,281 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:24,281 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:24,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:24,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:26,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:26,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:27,347 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:29,955 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:29,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:30,283 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:32,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:32,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:32,505 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:25:32,507 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:32,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:25:32,541 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:25:32,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:25:34,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:34,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:36,285 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:36,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:37,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:38,383 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:39,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:39,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:41,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:42,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:42,290 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:43,438 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:45,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:45,139 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:47,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:47,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:47,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:25:47,541 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:25:47,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:25:48,292 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:48,681 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:50,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:50,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:52,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:52,074 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:54,294 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:25:54,563 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:25:55,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:55,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:58,859 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:25:58,894 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:58,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:25:59,619 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:00,296 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:01,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:01,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:02,507 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:26:02,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:26:02,541 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:26:02,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:26:03,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:03,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:04,684 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:06,298 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:06,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:06,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:08,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:08,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:09,685 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:11,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:11,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:12,300 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:13,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:13,948 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:14,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:16,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:16,838 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:17,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:26:17,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:26:17,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:26:18,302 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:18,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:18,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:20,713 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:21,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:21,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:23,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:23,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:24,305 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:25,847 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:26,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:26,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:28,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:28,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:30,307 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:30,901 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:31,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:31,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:32,508 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:26:32,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:26:32,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:26:32,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:26:33,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:33,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:35,958 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:36,309 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:37,370 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:26:37,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:37,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:40,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:40,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:40,958 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:42,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:42,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:42,311 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:45,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:45,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:46,017 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:47,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:47,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:47,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:26:47,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:26:47,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:26:48,313 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:50,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:50,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:51,414 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:52,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:52,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:54,315 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:26:55,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:55,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:57,128 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:26:57,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:26:57,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:00,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:00,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:00,317 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:02,185 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:02,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:02,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:02,516 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:27:02,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:27:02,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:27:02,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:27:04,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:04,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:06,319 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:07,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:07,231 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:07,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:09,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:09,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:12,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:12,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:12,319 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:12,321 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:15,860 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:27:15,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:15,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:17,320 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:17,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:27:17,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:27:17,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:27:18,323 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:18,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:18,923 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:20,961 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:20,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:22,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:23,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:23,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:24,325 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:25,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:25,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:28,412 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:28,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:29,008 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:30,328 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:31,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:31,043 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:32,512 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:27:32,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:27:32,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:27:32,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:27:33,674 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:34,008 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:34,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:36,063 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:36,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:36,330 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:39,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:39,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:39,540 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:41,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:41,085 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:42,332 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:43,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:43,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:44,598 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:46,024 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:46,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:47,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:27:47,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:27:47,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:27:49,160 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:49,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:50,386 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:51,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:51,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:52,335 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:53,786 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:27:53,787 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:27:53,787 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:27:53,788 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:27:54,336 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:27:54,337 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:55,055 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:27:55,079 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:55,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:55,788 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:27:56,337 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:27:58,116 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:27:58,130 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:00,167 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:00,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:00,338 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:01,771 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:02,514 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:28:02,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:28:02,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:28:02,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:28:03,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:03,224 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:05,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:05,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:07,066 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:08,341 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:08,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:08,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:10,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:10,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:12,067 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:12,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:12,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:14,343 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:15,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:15,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:17,154 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:17,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:28:17,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:28:17,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:28:17,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:17,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:19,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:19,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:20,346 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:22,213 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:22,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:22,835 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:25,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:25,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:26,348 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:27,289 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:27,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:27,505 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:30,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:30,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:32,350 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:32,362 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:32,515 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:28:32,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:28:32,542 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:28:32,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:28:37,154 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:28:37,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:37,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:37,437 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:38,352 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:39,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:39,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:42,438 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:42,579 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:42,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:44,354 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:44,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:44,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:47,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:47,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:47,516 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:47,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:28:47,543 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:28:47,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:28:49,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:49,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:50,356 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:51,298 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:51,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:52,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:54,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:54,476 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:56,358 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:28:56,496 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:56,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:57,654 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:28:58,541 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:28:58,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:01,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:01,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:02,360 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:02,517 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:29:02,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:29:02,543 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:29:02,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:29:02,774 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:03,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:03,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:05,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:05,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:07,825 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:08,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:08,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:09,363 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:11,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:11,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:13,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:15,365 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:15,556 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:29:15,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:15,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:17,542 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:29:17,543 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:29:17,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:29:18,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:18,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:20,306 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:20,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:20,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:21,367 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:23,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:23,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:25,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:25,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:25,849 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:27,369 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:27,717 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:27,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:30,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:30,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:30,849 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:32,519 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:29:32,543 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:29:32,543 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:29:32,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:29:32,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:32,816 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:33,372 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:34,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:34,854 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:36,320 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:36,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:36,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:37,374 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:39,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:39,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:41,375 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:41,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:41,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:41,806 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:43,376 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:44,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:44,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:45,376 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:46,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:46,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:47,301 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:47,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:29:47,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:29:47,592 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:29:49,378 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:49,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:49,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:52,428 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:53,706 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:29:53,877 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:53,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:55,380 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:29:55,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:55,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:57,782 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:29:58,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:29:58,893 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:00,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:00,926 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:01,382 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:02,520 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:30:02,543 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:30:02,543 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:30:02,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:30:03,754 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:04,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:04,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:06,159 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:06,173 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:07,384 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:08,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:08,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:08,894 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:11,062 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:30:11,063 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:30:11,064 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:30:11,065 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:30:11,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:11,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:11,385 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:30:12,386 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:13,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:13,332 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:13,386 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:14,066 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:15,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:15,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:17,543 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:30:17,543 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:30:17,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:30:18,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:18,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:19,388 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:19,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:20,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:20,465 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:22,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:22,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:25,182 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:25,390 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:25,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:25,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:27,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:27,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:30,274 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:31,393 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:31,485 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:30:31,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:32,522 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:30:32,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:30:32,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:30:32,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:30:32,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:34,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:34,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:35,393 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:37,396 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:37,648 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:37,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:39,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:39,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:40,393 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:41,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:41,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:43,399 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:44,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:44,511 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:45,446 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:46,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:46,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:47,543 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:30:47,543 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:30:47,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:30:49,402 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:49,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:49,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:50,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:51,645 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:51,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:53,403 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:54,591 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:54,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:56,027 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:30:56,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:56,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:59,405 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:30:59,635 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:30:59,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:01,114 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:01,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:01,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:02,524 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:31:02,543 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:31:02,544 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:31:02,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:31:04,569 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:04,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:05,407 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:06,210 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:06,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:06,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:10,184 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:31:10,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:10,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:11,290 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:11,410 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:12,933 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:12,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:14,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:15,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:16,359 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:17,412 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:17,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:31:17,544 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:31:17,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:31:17,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:17,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:19,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:20,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:21,447 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:22,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:22,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:23,414 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:24,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:24,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:26,448 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:27,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:27,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:29,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:29,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:29,416 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:31,520 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:31,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:31,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:32,526 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:31:32,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:31:32,544 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:31:32,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:31:34,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:34,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:35,418 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:36,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:36,634 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:36,647 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:39,602 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:39,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:41,420 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:41,672 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:42,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:42,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:45,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:45,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:46,752 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:47,422 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:47,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:31:47,544 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:31:47,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:31:49,041 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:31:49,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:49,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:51,826 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:51,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:51,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:53,425 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:31:53,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:53,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:56,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:56,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:56,826 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:31:58,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:58,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:31:59,427 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:00,936 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:00,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:01,912 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:02,528 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:32:02,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:32:02,544 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:32:02,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:32:03,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:03,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:05,428 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:05,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:05,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:06,993 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:09,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:09,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:11,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:11,095 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:11,430 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:12,054 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:14,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:14,097 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:16,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:16,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:17,129 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:17,432 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:17,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:32:17,544 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:32:17,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:32:18,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:18,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:20,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:20,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:22,460 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:22,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:23,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:23,434 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:27,460 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:27,724 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:32:27,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:27,750 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:29,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:29,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:32,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:32,374 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:32,461 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:32,529 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:32:32,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:32:32,544 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:32:32,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:32:34,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:34,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:37,221 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:37,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:37,682 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:39,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:39,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:41,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:41,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:41,440 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:43,276 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:44,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:44,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:45,441 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:46,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:46,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:47,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:32:47,544 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:32:47,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:32:48,765 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:48,840 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:32:48,842 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:32:48,842 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:32:48,844 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:32:49,443 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:32:49,443 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:49,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:49,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:50,443 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:51,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:51,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:53,845 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:32:54,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:54,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:56,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:56,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:57,445 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:32:58,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:58,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:32:59,835 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:01,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:01,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:02,530 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:33:02,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:33:02,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:33:02,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:33:03,447 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:05,755 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:06,384 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:33:06,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:06,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:09,374 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:09,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:09,451 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:10,966 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:11,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:11,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:14,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:14,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:15,454 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:16,035 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:16,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:16,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:17,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:33:17,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:33:17,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:33:18,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:18,888 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:21,094 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:21,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:21,230 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:21,457 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:23,658 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:23,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:25,459 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:25,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:25,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:26,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:28,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:28,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:30,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:30,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:31,461 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:31,752 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:32,532 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:33:32,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:33:32,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:33:32,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:33:33,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:33,935 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:35,463 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:35,975 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:35,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:37,626 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:38,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:38,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:40,961 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:40,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:41,465 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:43,467 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:44,675 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:33:44,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:44,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:47,467 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:47,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:33:47,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:33:47,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:33:47,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:47,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:48,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:49,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:49,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:52,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:52,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:53,468 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:54,631 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:54,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:54,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:57,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:57,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:59,470 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:33:59,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:33:59,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:33:59,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:02,533 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:34:02,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:34:02,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:34:02,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:34:02,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:02,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:04,789 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:04,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:04,926 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:05,473 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:06,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:06,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:09,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:09,789 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:09,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:11,474 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:11,837 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:11,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:14,855 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:14,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:14,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:16,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:16,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:17,476 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:17,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:34:17,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:34:17,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:34:19,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:19,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:20,676 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:23,479 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:23,667 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:34:23,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:23,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:26,013 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:26,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:26,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:28,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:28,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:29,480 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:31,084 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:31,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:31,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:32,535 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:34:32,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:34:32,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:34:32,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:34:33,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:33,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:35,482 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:36,156 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:36,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:36,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:38,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:38,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:41,255 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:41,483 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:41,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:41,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:43,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:43,902 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:46,256 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:46,937 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:46,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:47,485 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:47,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:34:47,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:34:47,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:34:48,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:49,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:51,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:51,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:51,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:53,487 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:34:53,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:53,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:56,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:56,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:57,417 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:34:58,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:58,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:34:59,489 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:02,493 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:02,538 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:35:02,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:35:02,545 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:35:02,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:35:03,657 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:35:03,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:03,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:05,491 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:05,759 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:05,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:07,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:08,786 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:08,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:10,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:10,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:11,493 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:12,665 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:12,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:12,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:15,604 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:15,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:16,815 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:35:16,817 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:35:16,818 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:35:16,819 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:35:17,495 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:35:17,495 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:17,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:35:17,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:35:17,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:35:17,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:17,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:17,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:18,495 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:20,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:20,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:22,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:22,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:22,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:23,497 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:25,627 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:25,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:27,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:27,674 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:27,829 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:29,499 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:29,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:29,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:32,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:32,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:32,538 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:35:32,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:35:32,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:35:32,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:35:33,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:34,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:34,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:37,501 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:37,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:37,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:39,014 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:41,214 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:35:41,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:41,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:43,502 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:44,086 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:44,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:44,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:46,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:46,291 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:47,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:35:47,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:35:47,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:35:48,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:48,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:49,504 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:49,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:51,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:51,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:53,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:53,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:55,269 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:35:55,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:55,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:55,505 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:35:58,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:35:58,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:00,293 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:00,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:00,347 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:01,507 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:02,540 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:36:02,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:36:02,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:36:02,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:36:03,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:03,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:05,508 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:05,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:05,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:05,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:07,509 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:08,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:08,604 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:10,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:10,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:11,192 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:11,510 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:13,510 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:13,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:13,770 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:15,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:15,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:16,217 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:17,514 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:17,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:36:17,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:36:17,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:36:19,357 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:36:19,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:20,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:21,515 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:22,198 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:22,511 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:22,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:24,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:24,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:27,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:27,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:27,517 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:28,124 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:29,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:29,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:32,542 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:36:32,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:36:32,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:36:32,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:36:32,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:32,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:33,518 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:33,764 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:34,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:34,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:36,692 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:36,706 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:39,306 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:39,520 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:39,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:39,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:41,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:41,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:43,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:43,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:44,393 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:45,521 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:46,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:46,462 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:47,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:36:47,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:36:47,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:36:48,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:48,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:49,489 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:51,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:51,428 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:51,523 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:53,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:53,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:54,491 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:36:57,525 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:36:57,951 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:36:58,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:58,224 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:36:59,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:00,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:00,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:02,293 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:02,544 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:37:02,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:37:02,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:37:02,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:37:03,526 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:03,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:04,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:05,706 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:05,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:07,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:07,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:09,528 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:09,752 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:09,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:09,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:11,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:11,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:14,845 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:14,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:14,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:15,530 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:16,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:16,953 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:17,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:37:17,546 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:37:17,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:37:19,991 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:20,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:20,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:21,539 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:22,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:22,118 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:24,138 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:24,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:24,992 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:27,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:27,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:27,540 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:29,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:29,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:30,340 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:30,747 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:37:30,748 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:37:30,749 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:37:30,749 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:37:31,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:31,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:31,542 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:37:31,542 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:32,542 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:32,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:37:32,547 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:37:32,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:37:32,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:37:35,524 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:35,543 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:36,053 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:37:36,364 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:36,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:38,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:38,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:39,544 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:40,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:40,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:41,125 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:43,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:43,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:45,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:45,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:45,546 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:46,249 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:47,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:37:47,547 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:37:47,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:37:48,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:48,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:50,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:50,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:51,349 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:51,550 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:52,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:52,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:54,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:54,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:56,349 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:37:56,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:56,584 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:57,552 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:37:58,604 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:37:58,615 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:00,649 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:01,474 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:01,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:02,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:38:02,547 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:38:02,547 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:38:02,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:38:03,554 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:03,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:03,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:05,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:05,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:06,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:07,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:08,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:09,556 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:12,404 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:38:12,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:12,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:12,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:15,557 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:15,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:15,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:17,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:38:17,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:38:17,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:38:17,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:17,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:17,917 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:21,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:21,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:21,559 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:23,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:23,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:23,914 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:25,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:25,130 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:27,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:27,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:27,561 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:29,017 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:29,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:29,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:32,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:38:32,548 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:38:32,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:38:32,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:38:33,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:33,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:33,562 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:34,148 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:35,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:35,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:37,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:37,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:39,148 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:40,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:40,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:41,565 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:42,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:42,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:44,259 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:45,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:45,561 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:47,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:38:47,547 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:38:47,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:38:47,566 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:47,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:47,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:49,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:51,262 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:38:51,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:51,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:53,568 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:38:54,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:54,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:55,476 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:38:56,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:56,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:59,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:59,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:38:59,570 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:00,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:02,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:02,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:02,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:39:02,547 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:39:02,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:39:02,771 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:39:04,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:04,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:05,572 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:05,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:06,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:06,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:08,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:08,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:10,818 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:11,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:11,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:11,574 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:13,298 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:13,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:15,819 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:16,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:16,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:17,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:39:17,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:39:17,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:39:17,576 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:18,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:18,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:20,975 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:21,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:21,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:23,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:23,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:23,578 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:26,032 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:26,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:26,658 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:29,580 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:30,357 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:39:30,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:30,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:31,143 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:32,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:39:32,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:39:32,563 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:39:32,721 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:39:33,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:33,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:33,582 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:35,539 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:35,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:35,583 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:36,209 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:37,584 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:38,635 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:38,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:40,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:40,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:41,587 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:41,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:42,516 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:39:42,518 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:39:42,519 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:39:42,520 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:39:42,587 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:39:42,735 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:42,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:43,588 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:44,588 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:45,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:45,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:45,588 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:47,257 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:47,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:39:47,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:39:47,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:39:47,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:47,615 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:50,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:50,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:51,591 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:52,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:52,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:52,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:53,591 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:39:55,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:55,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:57,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:57,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:39:58,025 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:39:59,593 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:00,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:00,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:02,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:40:02,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:40:02,563 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:40:02,696 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:40:02,777 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:02,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:03,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:05,595 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:05,759 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:05,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:09,420 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:09,497 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:40:09,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:09,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:11,597 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:12,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:12,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:14,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:14,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:15,367 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:16,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:16,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:17,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:40:17,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:40:17,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:40:17,598 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:19,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:19,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:21,128 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:21,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:21,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:23,600 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:24,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:24,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:26,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:26,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:26,863 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:27,601 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:29,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:29,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:31,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:31,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:32,331 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:32,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:40:32,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:40:32,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:40:32,727 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:40:33,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:33,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:33,603 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:36,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:36,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:38,094 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:38,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:38,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:39,605 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:41,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:41,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:43,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:43,647 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:44,054 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:45,606 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:47,486 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:40:47,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:47,518 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:47,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:40:47,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:40:47,563 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:40:49,676 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:50,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:50,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:51,608 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:52,579 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:52,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:54,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:54,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:55,149 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:40:56,610 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:40:57,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:57,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:59,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:40:59,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:00,150 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:02,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:41:02,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:41:02,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:02,563 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:41:02,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:02,611 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:02,703 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:41:04,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:04,602 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:05,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:07,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:07,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:08,613 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:09,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:09,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:11,112 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:12,745 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:12,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:14,614 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:14,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:14,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:16,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:17,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:41:17,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:41:17,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:41:17,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:17,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:19,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:19,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:21,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:22,609 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:22,617 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:23,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:26,618 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:26,800 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:41:26,824 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:26,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:28,289 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:29,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:29,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:31,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:31,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:32,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:41:32,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:41:32,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:41:32,619 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:32,740 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:41:33,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:35,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:35,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:37,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:37,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:38,621 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:38,819 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:40,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:40,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:42,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:42,258 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:44,624 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:44,783 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:45,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:45,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:47,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:47,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:47,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:41:47,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:41:47,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:41:50,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:50,293 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:50,626 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:50,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:52,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:52,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:54,627 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:41:55,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:55,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:55,991 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:41:57,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:57,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:41:59,015 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:41:59,016 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:41:59,017 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:41:59,017 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:41:59,629 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:42:00,476 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:00,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:00,629 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:01,018 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:02,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:02,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:42:02,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:42:02,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:02,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:42:02,692 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:42:06,474 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:06,631 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:06,915 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:42:07,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:07,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:09,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:09,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:12,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:12,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:12,440 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:12,635 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:14,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:14,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:16,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:16,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:17,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:17,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:42:17,548 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:42:17,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:42:18,637 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:19,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:19,160 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:21,209 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:21,219 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:22,894 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:24,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:24,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:24,639 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:26,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:26,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:28,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:28,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:28,789 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:30,641 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:31,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:31,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:32,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:42:32,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:42:32,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:42:32,728 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:42:33,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:33,417 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:34,633 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:35,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:35,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:36,642 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:38,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:38,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:40,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:40,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:40,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:42,644 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:43,830 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:42:43,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:44,649 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:46,211 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:46,645 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:46,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:46,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:47,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:42:47,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:42:47,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:42:48,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:48,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:50,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:50,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:52,176 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:52,647 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:53,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:53,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:55,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:55,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:57,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:42:58,648 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:42:58,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:42:58,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:00,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:00,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:02,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:43:02,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:43:02,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:43:02,716 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:43:02,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:02,745 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:03,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:04,650 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:05,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:05,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:07,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:07,876 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:09,197 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:09,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:09,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:10,652 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:12,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:12,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:14,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:14,923 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:15,182 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:16,653 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:17,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:43:17,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:43:17,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:43:17,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:18,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:21,148 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:21,779 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:43:21,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:21,810 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:22,655 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:23,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:23,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:26,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:26,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:26,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:28,657 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:28,868 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:28,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:30,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:30,918 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:32,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:32,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:43:32,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:43:32,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:43:32,679 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:43:33,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:33,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:34,660 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:35,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:35,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:38,280 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:38,661 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:38,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:38,804 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:40,824 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:40,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:44,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:44,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:44,239 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:44,663 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:46,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:46,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:47,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:43:47,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:43:47,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:43:48,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:48,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:49,862 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:50,665 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:51,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:51,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:53,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:53,077 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:55,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:43:56,079 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:56,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:56,667 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:43:59,778 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:43:59,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:43:59,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:01,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:01,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:01,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:02,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:44:02,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:44:02,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:44:02,668 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:02,739 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:44:04,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:04,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:06,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:06,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:06,779 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:08,670 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:08,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:08,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:11,645 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:11,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:11,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:13,676 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:13,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:14,673 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:16,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:16,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:17,548 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:17,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:44:17,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:44:17,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:44:18,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:18,786 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:21,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:21,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:22,669 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:22,676 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:23,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:23,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:24,298 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:44:24,298 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:44:24,299 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:44:24,299 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:44:24,677 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:44:24,677 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:26,678 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:26,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:26,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:28,301 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:28,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:28,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:30,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:31,095 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:32,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:44:32,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:44:32,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:44:32,681 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:32,743 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:44:33,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:33,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:33,744 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:37,390 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:44:37,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:37,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:38,682 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:38,769 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:40,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:40,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:42,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:42,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:42,684 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:44,367 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:44,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:44,411 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:47,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:47,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:47,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:44:47,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:44:47,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:44:48,685 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:49,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:49,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:50,401 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:51,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:51,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:54,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:54,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:54,688 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:44:56,049 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:44:56,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:56,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:59,209 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:44:59,232 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:00,690 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:01,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:01,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:01,886 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:02,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:45:02,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:45:02,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:45:02,780 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:45:04,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:04,132 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:06,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:06,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:06,691 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:07,839 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:09,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:09,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:11,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:11,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:12,693 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:13,387 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:15,058 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:45:15,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:15,095 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:17,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:45:17,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:45:17,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:45:18,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:18,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:18,695 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:18,752 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:20,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:20,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:22,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:22,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:23,913 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:24,697 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:24,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:24,888 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:26,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:26,921 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:29,872 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:29,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:29,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:30,699 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:31,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:31,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:32,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:45:32,549 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:45:32,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:45:32,759 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:45:34,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:34,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:34,701 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:35,390 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:37,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:37,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:39,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:39,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:40,394 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:40,703 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:41,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:41,077 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:43,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:43,903 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:45,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:45,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:46,359 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:46,705 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:47,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:45:47,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:45:47,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:45:47,977 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:47,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:51,472 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:45:51,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:51,511 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:51,917 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:52,706 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:45:54,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:54,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:56,391 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:56,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:57,880 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:45:58,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:58,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:45:58,708 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:01,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:01,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:02,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:46:02,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:46:02,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:46:02,722 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:46:03,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:03,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:03,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:04,710 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:06,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:06,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:08,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:08,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:09,420 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:10,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:10,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:10,711 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:13,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:13,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:15,004 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:15,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:15,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:16,713 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:17,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:46:17,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:46:17,557 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:17,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:46:17,569 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:20,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:20,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:20,995 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:22,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:22,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:22,715 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:25,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:25,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:26,802 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:28,716 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:29,415 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:46:29,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:29,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:32,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:32,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:32,496 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:32,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:46:32,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:46:32,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:46:32,706 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:46:32,718 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:34,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:34,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:37,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:37,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:38,457 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:38,720 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:39,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:39,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:41,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:41,710 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:44,060 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:44,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:44,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:44,721 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:46,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:46,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:47,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:46:47,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:46:47,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:46:48,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:48,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:49,081 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:46:51,043 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:46:51,043 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:51,043 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:46:51,044 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:46:51,724 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:46:51,821 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:51,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:52,724 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:53,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:53,860 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:55,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:55,893 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:56,603 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:46:56,725 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:46:58,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:46:58,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:00,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:00,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:02,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:02,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:47:02,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:47:02,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:47:02,727 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:02,763 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:47:03,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:03,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:07,519 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:47:07,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:07,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:08,135 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:08,730 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:10,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:10,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:12,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:12,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:14,129 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:14,731 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:15,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:15,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:17,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:17,489 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:17,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:47:17,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:47:17,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:47:20,084 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:20,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:20,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:20,733 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:22,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:22,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:24,734 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:25,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:25,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:25,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:27,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:27,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:30,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:30,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:30,737 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:31,659 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:32,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:47:32,550 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:47:32,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:47:32,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:32,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:32,762 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:47:34,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:34,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:36,739 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:37,118 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:37,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:37,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:39,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:39,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:42,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:42,477 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:42,741 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:43,075 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:46,422 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:47:46,445 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:46,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:47,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:47:47,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:47:47,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:47:48,743 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:48,767 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:49,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:49,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:51,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:51,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:54,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:54,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:54,643 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:47:54,744 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:47:56,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:56,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:59,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:47:59,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:00,439 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:00,746 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:01,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:01,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:02,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:48:02,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:48:02,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:48:02,728 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:48:03,414 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:03,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:06,316 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:06,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:06,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:06,748 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:08,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:08,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:10,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:10,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:12,274 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:12,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:12,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:12,750 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:15,430 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:15,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:17,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:17,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:17,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:17,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:48:17,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:48:17,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:48:18,751 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:20,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:20,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:22,753 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:22,831 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:24,299 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:48:24,322 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:24,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:26,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:26,370 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:28,373 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:28,755 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:29,325 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:29,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:31,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:31,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:32,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:48:32,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:48:32,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:48:32,757 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:48:33,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:33,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:34,322 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:34,757 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:36,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:36,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:38,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:38,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:39,880 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:40,758 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:41,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:41,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:43,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:43,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:45,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:45,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:45,863 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:46,760 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:47,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:48:47,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:48:47,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:48:48,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:48,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:50,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:50,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:51,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:52,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:52,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:52,762 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:48:55,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:55,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:57,403 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:48:57,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:57,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:48:58,763 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:01,121 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:49:01,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:01,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:02,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:02,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:49:02,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:49:02,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:49:02,755 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:49:04,071 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:04,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:04,765 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:06,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:06,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:07,757 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:08,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:08,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:08,766 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:11,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:11,183 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:12,928 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:13,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:13,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:13,956 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:49:13,957 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:49:13,957 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:49:13,959 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:49:14,768 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:49:14,769 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:15,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:15,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:15,769 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:17,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:49:17,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:49:17,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:49:18,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:18,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:18,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:20,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:20,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:20,770 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:22,221 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:22,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:24,479 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:25,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:25,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:26,772 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:27,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:27,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:30,326 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:30,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:30,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:32,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:32,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:32,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:49:32,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:49:32,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:49:32,686 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:49:32,774 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:35,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:35,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:35,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:38,775 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:39,316 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:49:39,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:39,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:40,863 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:41,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:41,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:44,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:44,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:44,777 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:46,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:46,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:46,825 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:47,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:49:47,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:49:47,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:49:48,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:49,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:50,779 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:51,110 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:51,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:52,431 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:53,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:53,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:55,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:55,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:56,781 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:49:57,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:57,275 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:58,381 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:49:59,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:49:59,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:00,782 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:01,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:01,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:02,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:50:02,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:50:02,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:50:02,704 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:50:03,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:03,414 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:03,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:05,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:05,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:06,784 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:08,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:08,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:08,788 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:10,369 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:10,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:12,786 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:14,305 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:50:14,316 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:14,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:14,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:16,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:16,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:17,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:50:17,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:50:17,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:50:18,788 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:19,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:19,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:19,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:21,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:21,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:23,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:23,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:24,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:24,796 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:26,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:26,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:28,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:28,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:30,364 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:30,797 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:31,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:31,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:32,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:50:32,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:50:32,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:50:32,711 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:50:33,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:33,462 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:35,510 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:36,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:36,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:36,798 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:38,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:38,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:40,800 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:41,048 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:41,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:41,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:43,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:43,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:46,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:46,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:46,802 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:47,004 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:47,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:50:47,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:50:47,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:50:48,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:48,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:52,528 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:52,805 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:50:53,461 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:50:53,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:53,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:55,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:55,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:58,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:58,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:50:58,456 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:50:58,807 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:00,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:00,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:02,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:51:02,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:51:02,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:51:02,740 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:51:03,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:03,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:04,411 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:04,809 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:05,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:05,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:07,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:07,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:09,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:10,034 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:10,811 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:10,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:12,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:12,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:14,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:15,991 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:16,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:16,813 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:17,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:51:17,551 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:51:17,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:51:18,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:18,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:20,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:20,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:21,554 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:22,814 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:22,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:23,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:25,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:25,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:27,518 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:28,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:28,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:28,816 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:31,903 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:51:31,923 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:31,937 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:32,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:32,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:51:32,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:51:32,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:51:32,739 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:51:32,818 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:33,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:33,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:36,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:36,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:37,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:38,088 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:51:38,090 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:51:38,090 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:51:38,090 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:51:38,819 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:51:38,819 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:39,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:39,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:39,819 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:41,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:41,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:43,091 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:43,934 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:43,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:44,821 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:45,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:46,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:47,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:51:47,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:51:47,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:51:48,765 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:49,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:49,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:50,823 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:51,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:51,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:53,137 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:53,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:54,499 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:51:56,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:56,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:56,825 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:51:58,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:58,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:51:59,671 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:00,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:00,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:02,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:52:02,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:52:02,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:52:02,770 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:52:02,826 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:03,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:03,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:04,771 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:05,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:05,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:08,828 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:08,840 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:52:08,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:08,877 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:10,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:11,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:11,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:13,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:13,692 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:14,830 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:15,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:15,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:16,397 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:17,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:52:17,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:52:17,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:52:18,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:18,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:20,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:20,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:21,922 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:22,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:22,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:22,833 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:25,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:25,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:26,834 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:27,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:27,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:27,823 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:29,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:29,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:32,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:52:32,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:52:32,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:52:32,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:32,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:32,763 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:52:32,835 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:33,764 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:34,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:34,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:36,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:36,748 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:38,838 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:39,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:39,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:39,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:41,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:41,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:44,839 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:45,190 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:45,511 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:52:45,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:45,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:47,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:52:47,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:52:47,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:52:48,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:48,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:50,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:50,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:50,841 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:51,149 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:52,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:52,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:55,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:55,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:56,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:52:56,843 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:52:57,625 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:52:57,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:00,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:00,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:02,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:02,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:53:02,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:53:02,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:53:02,773 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:53:02,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:02,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:02,845 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:53:04,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:04,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:07,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:07,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:08,322 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:08,846 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:53:09,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:09,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:11,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:11,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:12,848 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:53:14,217 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:14,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:14,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:16,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:16,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:17,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:53:17,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:53:17,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:53:18,850 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:53:18,993 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:20,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:20,189 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:23,708 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:53:23,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:23,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:24,851 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:53:25,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:25,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:25,810 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:28,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:28,674 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:30,692 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:30,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:30,854 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:53:31,022 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:32,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:53:32,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:53:32,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:53:32,730 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:53:33,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:33,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:35,786 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:35,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:36,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:38,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:38,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:40,757 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:40,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:41,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:42,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:43,604 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:45,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:45,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:46,859 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:53:47,267 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:47,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:53:47,552 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:53:47,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:53:47,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:47,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:50,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:50,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:52,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:52,611 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:52,862 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:53:53,216 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:55,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:55,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:57,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:57,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:53:58,803 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:53:58,863 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:01,231 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:54:01,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:01,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:02,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:54:02,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:54:02,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:54:02,737 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:54:04,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:04,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:04,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:04,866 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:06,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:06,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:09,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:09,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:10,496 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:10,868 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:11,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:11,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:14,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:14,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:15,378 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:54:15,380 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:54:15,380 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:54:15,381 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:54:15,869 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:54:16,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:16,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:16,381 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:16,870 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:17,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:54:17,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:54:17,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:54:17,870 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:19,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:19,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:21,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:21,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:22,251 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:22,872 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:24,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:24,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:26,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:26,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:27,796 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:28,873 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:29,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:29,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:31,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:31,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:32,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:54:32,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:54:32,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:54:32,693 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:54:33,756 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:33,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:33,953 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:34,875 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:36,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:36,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:38,877 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:39,226 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:40,656 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:54:40,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:40,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:43,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:43,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:44,880 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:45,025 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:45,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:45,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:47,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:54:47,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:54:47,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:54:47,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:47,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:50,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:50,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:50,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:50,881 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:52,845 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:52,854 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:54,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:54,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:56,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:54:56,883 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:54:58,017 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:54:58,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:00,065 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:00,074 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:02,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:02,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:02,501 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:02,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:55:02,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:55:02,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:55:02,672 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:55:02,885 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:05,127 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:05,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:07,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:07,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:07,910 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:08,887 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:10,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:10,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:12,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:12,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:13,874 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:14,888 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:15,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:15,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:17,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:55:17,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:55:17,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:55:19,278 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:55:19,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:19,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:20,463 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:21,332 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:21,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:22,891 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:23,369 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:23,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:24,891 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:26,296 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:26,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:26,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:28,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:28,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:30,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:30,445 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:30,893 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:31,933 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:32,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:55:32,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:55:32,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:55:32,762 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:55:33,137 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:33,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:35,165 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:35,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:36,894 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:37,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:37,219 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:37,864 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:40,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:40,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:42,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:42,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:42,902 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:43,828 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:44,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:44,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:47,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:47,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:47,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:55:47,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:55:47,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:55:48,903 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:49,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:49,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:49,713 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:51,275 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:51,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:54,831 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:55:54,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:54,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:54,905 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:55:55,406 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:55:57,674 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:57,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:59,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:55:59,748 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:00,907 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:00,954 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:01,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:01,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:02,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:56:02,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:56:02,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:56:02,692 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:56:04,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:04,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:06,908 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:06,919 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:06,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:06,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:08,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:08,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:11,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:11,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:12,507 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:12,910 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:14,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:14,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:16,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:16,074 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:16,911 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:17,552 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:17,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:56:17,553 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:56:17,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:56:19,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:19,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:21,197 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:21,207 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:22,913 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:23,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:23,271 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:23,458 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:26,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:26,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:28,421 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:28,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:28,914 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:28,925 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:32,141 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:56:32,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:32,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:32,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:56:32,554 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:56:32,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:56:32,704 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:56:34,880 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:34,916 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:35,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:35,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:37,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:37,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:39,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:39,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:39,439 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:56:39,440 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:56:39,440 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:56:39,441 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:56:39,917 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:56:40,441 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:40,918 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:41,918 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:42,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:42,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:44,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:44,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:46,331 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:46,919 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:47,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:47,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:47,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:56:47,554 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:56:47,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:56:49,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:49,541 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:51,956 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:52,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:52,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:52,921 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:54,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:54,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:57,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:57,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:57,911 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:56:58,923 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:56:59,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:56:59,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:02,553 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:57:02,554 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:57:02,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:02,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:57:02,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:02,769 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:57:03,770 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:04,627 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:04,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:04,926 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:06,669 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:06,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:08,927 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:09,493 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:10,206 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:57:10,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:10,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:13,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:13,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:14,929 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:15,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:15,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:15,453 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:17,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:57:17,556 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:57:17,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:57:18,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:18,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:20,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:20,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:20,932 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:21,030 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:22,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:22,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:24,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:24,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:26,933 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:26,995 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:27,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:27,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:29,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:29,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:32,325 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:32,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:32,554 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:32,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:57:32,555 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:57:32,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:57:32,724 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:57:32,935 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:34,366 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:34,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:37,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:37,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:37,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:38,937 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:39,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:39,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:42,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:42,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:42,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:44,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:44,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:44,939 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:47,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:57:47,555 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:57:47,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:57:47,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:48,047 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:57:48,071 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:48,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:50,941 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:51,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:51,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:53,097 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:57:53,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:53,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:55,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:55,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:56,942 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:57:58,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:58,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:57:58,968 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:00,541 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:00,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:02,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:58:02,555 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:58:02,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:58:02,745 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:58:02,944 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:03,606 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:03,627 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:04,746 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:05,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:05,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:06,945 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:08,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:08,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:10,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:10,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:10,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:12,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:12,893 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:12,947 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:15,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:16,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:16,171 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:17,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:58:17,555 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:58:17,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:58:18,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:18,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:18,949 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:21,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:21,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:22,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:23,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:23,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:24,951 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:27,049 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:58:27,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:27,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:27,754 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:29,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:30,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:30,952 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:32,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:32,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:32,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:58:32,555 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:58:32,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:58:32,756 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:58:32,756 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:34,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:34,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:36,955 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:37,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:37,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:38,346 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:39,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:39,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:41,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:41,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:42,957 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:44,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:44,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:44,289 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:46,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:46,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:47,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:58:47,556 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:58:47,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:58:48,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:48,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:48,958 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:49,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:51,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:51,414 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:53,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:53,443 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:54,840 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:58:55,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:55,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:55,961 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:58:58,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:58:58,209 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:00,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:00,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:00,798 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:01,962 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:02,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:59:02,556 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:59:02,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:59:02,780 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:59:04,066 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:59:04,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:04,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:04,459 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 14:59:04,461 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 14:59:04,461 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 14:59:04,461 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 14:59:04,963 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 14:59:05,963 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:06,462 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:07,099 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:07,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:09,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:09,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:11,965 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:12,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:12,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:12,346 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:14,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:14,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:16,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:16,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:17,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:17,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:59:17,556 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:59:17,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:59:17,967 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:19,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:19,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:21,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:21,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:22,935 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:23,969 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:24,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:24,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:26,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:26,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:28,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:28,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:28,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:29,970 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:31,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:31,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:32,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:59:32,556 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:59:32,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:59:32,717 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 14:59:33,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:33,627 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:34,491 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:35,653 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:35,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:35,972 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:38,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:38,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:40,453 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:41,974 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:42,366 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 14:59:42,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:42,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:44,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:44,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:46,044 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:46,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:46,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:47,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 14:59:47,556 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 14:59:47,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 14:59:47,976 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:49,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:49,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:51,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:51,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:52,008 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:53,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:53,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:53,978 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:56,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:56,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:57,625 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 14:59:57,979 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 14:59:58,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 14:59:58,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:01,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:01,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:02,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:00:02,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:00:02,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:00:02,775 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:00:02,778 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:03,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:03,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:05,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:05,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:05,982 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:07,777 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:08,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:08,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:09,983 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:10,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:10,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:12,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:12,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:13,168 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:15,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:15,902 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:15,985 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:17,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:00:17,557 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:00:17,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:00:18,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:19,532 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:00:19,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:19,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:21,986 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:22,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:22,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:24,131 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:24,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:24,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:26,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:26,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:27,988 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:29,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:29,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:29,828 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:31,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:31,588 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:32,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:00:32,557 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:00:32,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:00:32,709 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:00:33,990 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:34,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:34,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:35,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:36,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:36,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:39,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:39,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:39,994 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:41,381 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:41,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:41,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:44,779 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:44,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:45,997 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:46,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:46,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:47,338 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:47,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:00:47,557 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:00:47,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:00:49,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:49,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:51,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:51,999 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:52,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:52,682 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:55,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:55,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:56,000 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:00:58,508 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:00:58,806 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:00:58,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:00:58,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:00,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:00,888 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:02,002 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:02,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:01:02,557 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:01:02,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:01:02,720 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:01:03,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:03,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:03,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:05,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:05,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:08,005 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:08,827 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:08,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:08,924 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:10,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:10,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:13,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:13,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:14,006 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:14,090 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:15,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:15,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:16,007 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:17,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:01:17,624 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:01:17,625 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:01:19,099 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:19,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:19,623 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:19,650 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:01:19,651 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:01:19,652 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:01:19,652 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:01:20,008 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:01:20,009 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:21,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:21,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:22,009 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:23,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:23,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:24,010 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:25,134 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:25,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:26,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:28,011 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:28,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:28,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:30,012 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:30,898 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:31,816 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:31,837 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:32,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:01:32,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:01:32,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:01:32,761 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:01:33,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:33,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:34,013 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:36,762 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:37,589 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:01:37,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:37,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:39,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:39,676 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:40,015 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:42,342 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:42,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:42,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:44,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:44,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:46,017 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:46,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:46,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:47,476 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:47,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:01:47,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:01:47,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:01:49,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:49,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:51,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:51,669 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:52,897 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:53,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:53,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:54,019 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:01:56,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:56,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:58,015 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:01:58,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:01:58,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:00,021 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:01,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:01,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:02,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:02:02,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:02:02,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:02:02,764 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:02:03,131 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:04,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:04,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:06,023 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:06,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:06,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:08,132 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:08,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:08,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:11,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:11,094 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:12,025 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:13,274 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:14,933 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:02:14,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:14,968 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:16,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:16,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:17,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:02:17,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:02:17,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:02:18,027 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:18,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:19,024 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:19,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:22,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:22,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:24,029 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:24,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:24,275 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:24,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:26,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:26,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:29,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:29,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:29,676 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:30,031 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:31,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:31,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:32,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:02:32,559 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:02:32,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:02:32,707 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:02:33,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:33,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:34,812 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:36,033 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:36,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:36,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:38,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:38,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:39,812 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:41,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:41,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:42,034 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:43,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:43,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:44,957 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:45,595 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:45,611 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:47,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:02:47,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:02:47,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:02:48,036 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:48,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:48,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:50,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:52,724 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:02:52,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:52,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:54,038 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:02:54,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:54,786 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:56,231 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:02:57,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:02:58,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:00,040 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:00,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:00,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:01,374 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:02,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:02,109 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:02,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:03:02,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:03:02,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:03:02,755 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:03:05,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:05,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:06,042 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:06,508 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:07,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:07,222 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:09,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:09,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:11,643 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:12,043 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:12,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:12,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:14,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:14,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:16,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:16,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:16,644 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:17,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:03:17,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:03:17,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:03:18,045 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:19,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:19,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:21,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:21,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:21,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:23,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:23,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:24,047 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:26,923 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:27,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:27,058 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:30,798 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:03:30,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:30,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:32,050 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:32,060 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:32,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:03:32,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:03:32,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:03:32,755 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:03:33,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:34,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:36,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:36,054 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:37,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:38,052 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:38,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:38,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:41,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:41,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:41,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:03:41,767 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:03:41,768 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:03:41,769 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:03:42,053 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:03:42,770 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:43,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:43,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:44,054 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:45,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:45,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:47,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:03:47,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:03:47,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:03:48,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:48,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:50,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:50,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:50,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:52,056 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:52,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:52,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:55,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:55,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:55,962 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:03:56,057 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:03:57,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:57,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:59,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:03:59,821 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:01,114 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:02,058 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:02,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:04:02,558 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:04:02,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:04:02,750 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:04:02,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:02,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:04,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:04,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:06,258 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:08,060 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:08,611 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:04:08,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:08,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:11,259 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:11,648 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:11,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:13,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:13,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:14,063 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:15,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:15,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:16,394 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:17,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:04:17,559 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:04:17,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:04:18,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:18,717 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:20,065 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:20,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:20,748 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:21,543 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:22,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:22,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:25,710 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:25,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:26,068 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:26,673 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:27,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:27,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:30,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:30,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:31,824 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:32,072 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:32,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:04:32,559 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:04:32,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:04:32,743 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:04:32,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:32,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:35,948 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:35,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:37,344 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:37,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:38,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:38,073 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:40,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:40,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:40,074 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:42,075 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:42,886 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:43,071 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:43,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:46,076 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:46,839 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:04:46,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:46,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:47,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:04:47,559 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:04:47,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:04:48,077 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:48,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:49,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:49,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:52,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:52,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:53,889 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:04:54,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:54,063 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:54,079 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:04:56,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:56,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:58,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:58,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:04:58,890 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:00,081 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:01,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:01,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:02,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:05:02,559 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:05:02,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:05:02,730 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:05:03,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:03,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:04,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:06,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:06,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:08,084 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:09,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:09,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:10,145 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:11,052 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:11,065 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:12,085 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:14,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:14,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:15,293 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:16,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:16,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:17,558 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:05:17,559 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:05:17,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:05:18,087 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:18,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:18,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:20,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:21,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:21,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:24,089 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:25,019 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:05:25,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:25,052 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:26,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:27,076 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:27,085 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:29,112 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:29,127 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:31,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:31,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:31,888 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:32,091 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:32,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:05:32,559 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:05:32,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:05:32,766 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:05:33,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:33,923 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:35,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:35,961 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:36,836 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:38,093 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:39,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:39,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:41,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:41,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:41,837 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:43,120 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:43,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:44,095 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:46,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:46,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:46,969 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:47,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:05:47,559 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:05:47,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:05:48,224 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:48,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:50,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:50,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:52,097 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:52,776 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:05:53,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:53,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:55,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:55,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:56,099 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:05:57,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:57,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:05:58,256 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:00,478 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:06:00,479 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:06:00,479 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:06:00,479 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:06:01,100 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:06:01,652 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:06:02,100 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:02,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:02,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:02,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:06:02,560 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:06:02,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:06:02,737 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:06:03,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:04,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:04,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:06,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:06,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:08,103 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:09,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:09,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:09,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:11,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:11,414 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:13,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:13,445 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:14,105 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:14,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:16,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:16,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:17,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:06:17,560 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:06:17,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:06:18,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:18,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:20,519 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:20,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:20,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:22,107 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:23,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:23,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:25,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:25,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:25,972 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:26,109 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:27,755 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:27,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:30,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:30,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:30,973 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:32,110 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:32,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:06:32,560 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:06:32,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:06:32,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:32,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:32,702 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:06:34,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:34,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:36,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:38,112 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:39,145 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:06:39,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:39,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:41,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:41,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:42,399 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:43,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:43,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:44,114 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:46,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:46,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:47,537 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:47,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:06:47,560 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:06:47,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:06:48,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:48,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:50,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:50,935 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:52,117 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:53,143 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:06:54,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:54,071 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:56,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:56,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:58,118 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:06:58,153 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:58,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:06:58,802 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:01,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:01,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:02,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:07:02,560 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:07:02,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:07:02,712 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:07:03,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:03,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:04,120 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:04,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:05,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:05,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:08,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:08,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:10,050 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:10,122 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:10,485 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:10,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:12,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:12,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:15,196 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:16,101 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:07:16,124 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:16,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:16,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:17,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:07:17,560 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:07:17,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:07:19,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:19,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:20,326 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:21,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:21,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:22,125 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:23,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:23,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:25,327 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:26,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:26,467 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:28,127 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:28,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:28,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:30,467 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:31,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:31,770 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:32,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:07:32,560 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:07:32,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:07:32,784 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:07:33,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:33,817 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:34,129 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:35,786 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:35,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:35,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:38,935 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:38,960 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:40,131 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:40,914 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:40,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:41,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:43,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:43,085 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:46,045 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:46,132 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:46,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:46,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:47,560 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:07:47,560 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:07:47,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:07:48,291 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:48,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:50,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:50,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:51,193 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:52,134 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:54,549 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:07:55,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:55,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:56,194 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:07:57,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:57,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:58,137 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:07:59,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:07:59,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:01,312 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:02,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:02,221 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:02,560 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:08:02,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:08:02,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:08:02,738 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:08:04,140 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:04,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:04,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:06,440 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:07,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:07,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:09,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:09,424 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:10,142 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:11,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:12,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:12,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:14,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:14,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:16,144 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:16,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:16,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:16,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:17,560 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:08:17,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:08:17,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:08:19,430 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:19,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:21,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:21,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:21,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:23,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:23,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:24,147 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:26,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:26,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:26,870 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:28,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:28,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:30,149 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:32,001 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:32,350 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:08:32,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:32,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:32,560 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:08:32,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:08:32,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:08:32,771 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:08:34,207 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:08:34,208 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:08:34,209 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:08:34,209 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:08:35,151 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:08:35,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:35,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:36,151 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:37,151 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:37,210 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:37,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:37,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:40,489 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:40,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:42,153 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:42,304 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:42,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:42,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:44,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:44,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:47,446 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:47,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:47,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:47,560 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:08:47,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:08:47,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:08:48,155 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:49,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:49,557 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:52,496 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:52,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:52,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:54,157 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:08:54,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:54,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:57,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:57,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:08:57,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:59,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:08:59,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:00,158 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:02,560 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:09:02,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:09:02,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:09:02,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:02,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:02,785 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:09:02,785 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:04,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:04,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:06,160 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:07,870 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:07,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:08,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:11,942 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:09:11,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:11,993 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:12,163 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:13,006 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:14,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:14,024 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:17,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:17,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:17,560 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:09:17,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:09:17,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:09:18,165 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:18,770 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:19,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:19,239 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:21,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:21,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:24,167 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:24,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:24,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:24,415 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:26,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:26,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:28,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:28,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:29,415 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:30,169 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:31,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:31,692 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:32,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:09:32,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:09:32,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:09:32,778 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:09:33,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:33,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:34,780 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:35,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:35,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:38,171 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:38,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:38,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:40,114 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:40,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:40,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:42,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:42,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:44,174 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:45,499 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:45,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:45,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:47,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:09:47,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:09:47,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:09:49,499 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:09:49,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:49,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:51,213 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:51,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:51,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:52,176 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:54,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:54,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:56,178 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:09:56,305 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:09:56,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:56,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:58,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:09:58,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:01,306 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:01,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:01,937 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:02,180 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:02,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:10:02,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:10:02,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:10:02,766 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:10:03,975 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:04,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:06,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:06,077 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:06,767 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:08,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:08,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:10,182 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:10,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:10,934 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:12,054 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:12,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:13,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:16,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:16,184 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:16,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:17,448 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:17,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:10:17,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:10:17,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:10:18,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:18,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:20,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:20,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:22,186 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:22,844 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:23,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:23,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:27,102 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:10:27,123 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:27,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:28,188 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:28,238 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:29,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:29,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:32,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:32,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:32,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:10:32,561 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:10:32,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:10:32,763 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:10:33,393 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:34,190 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:34,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:34,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:36,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:36,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:38,394 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:39,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:39,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:40,192 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:41,424 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:41,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:43,530 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:44,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:44,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:46,194 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:46,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:46,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:47,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:10:47,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:10:47,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:10:48,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:48,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:51,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:51,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:51,968 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:52,196 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:54,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:54,011 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:56,041 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:56,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:56,843 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:10:58,199 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:10:59,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:10:59,123 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:01,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:01,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:01,994 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:02,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:11:02,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:11:02,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:11:02,697 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:11:04,201 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:04,927 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:11:04,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:04,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:06,218 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:11:06,219 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:11:06,219 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:11:06,220 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:11:07,202 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:11:07,220 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:08,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:08,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:08,202 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:09,202 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:10,183 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:10,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:10,203 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:12,203 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:12,219 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:12,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:12,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:15,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:15,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:16,205 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:17,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:17,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:17,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:11:17,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:11:17,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:11:18,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:19,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:19,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:22,206 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:22,391 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:22,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:24,141 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:24,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:24,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:26,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:26,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:28,208 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:29,307 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:29,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:29,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:31,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:31,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:32,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:11:32,563 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:11:32,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:11:32,762 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:11:33,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:33,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:34,210 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:34,763 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:36,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:36,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:38,789 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:38,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:40,212 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:40,630 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:42,685 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:11:42,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:42,717 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:45,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:45,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:45,785 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:46,214 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:47,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:11:47,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:11:47,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:11:47,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:47,595 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:49,615 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:49,625 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:50,928 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:52,216 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:52,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:52,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:54,768 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:54,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:55,929 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:11:56,817 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:56,827 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:58,218 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:11:59,975 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:11:59,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:01,083 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:02,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:02,079 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:02,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:12:02,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:12:02,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:12:02,706 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:12:04,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:04,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:04,220 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:06,225 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:07,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:07,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:09,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:09,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:10,223 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:11,160 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:11,173 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:11,373 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:14,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:14,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:16,226 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:16,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:16,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:16,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:17,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:12:17,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:12:17,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:12:19,942 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:12:19,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:19,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:21,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:22,228 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:22,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:22,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:24,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:24,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:26,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:28,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:28,054 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:28,230 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:30,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:30,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:31,983 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:32,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:32,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:32,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:12:32,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:12:32,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:12:32,717 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:12:35,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:35,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:36,233 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:37,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:37,215 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:37,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:40,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:40,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:42,235 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:42,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:42,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:42,473 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:45,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:45,477 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:47,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:47,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:47,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:12:47,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:47,562 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:12:47,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:12:48,238 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:49,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:49,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:52,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:52,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:52,786 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:54,240 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:12:54,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:54,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:58,016 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:12:58,308 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:12:58,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:12:58,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:00,242 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:01,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:01,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:02,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:13:02,563 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:13:02,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:13:02,730 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:13:03,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:03,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:03,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:05,391 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:05,401 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:06,246 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:08,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:08,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:09,358 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:10,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:10,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:12,248 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:12,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:12,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:14,501 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:15,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:15,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:17,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:17,307 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:17,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:13:17,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:13:17,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:13:20,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:20,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:20,638 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:22,254 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:22,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:22,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:24,255 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:25,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:25,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:25,756 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:27,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:27,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:28,664 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:13:28,665 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:13:28,666 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:13:28,666 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:13:29,257 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:13:30,257 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:30,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:30,653 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:31,257 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:31,667 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:32,258 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:32,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:13:32,565 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:13:32,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:13:32,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:32,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:32,761 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:13:36,479 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:13:36,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:36,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:36,763 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:38,260 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:39,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:39,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:41,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:41,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:42,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:43,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:43,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:46,263 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:46,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:46,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:47,397 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:47,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:13:47,565 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:13:47,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:13:48,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:48,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:50,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:50,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:52,265 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:52,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:54,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:54,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:56,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:56,097 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:58,009 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:13:58,267 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:13:59,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:13:59,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:01,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:01,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:02,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:14:02,565 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:14:02,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:14:02,762 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:14:03,254 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:04,269 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:04,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:04,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:06,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:06,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:08,255 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:08,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:08,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:11,272 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:11,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:11,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:13,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:15,304 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:14:15,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:15,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:17,274 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:17,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:17,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:17,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:14:17,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:14:17,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:14:20,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:20,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:21,069 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:22,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:22,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:23,276 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:24,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:24,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:26,141 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:27,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:27,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:29,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:29,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:31,279 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:31,358 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:31,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:31,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:32,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:14:32,565 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:14:32,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:14:32,765 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:14:34,627 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:34,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:36,523 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:36,674 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:36,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:37,281 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:38,710 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:38,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:41,669 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:41,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:41,860 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:43,283 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:43,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:43,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:45,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:45,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:46,670 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:47,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:14:47,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:14:47,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:14:48,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:48,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:49,286 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:52,643 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:52,733 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:14:52,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:52,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:54,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:54,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:55,287 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:14:57,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:57,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:58,134 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:14:59,882 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:14:59,894 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:01,289 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:15:01,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:01,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:02,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:15:02,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:15:02,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:15:02,703 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:15:03,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:04,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:05,011 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:07,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:07,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:07,291 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:15:08,970 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:09,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:09,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:11,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:11,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:13,971 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:14,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:14,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:16,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:16,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:17,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:15:17,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:15:17,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:15:20,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:20,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:20,817 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:22,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:22,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:23,296 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:15:24,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:24,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:26,001 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:27,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:27,539 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:29,298 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:15:31,296 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:15:31,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:31,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:31,618 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:32,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:15:32,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:15:32,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:15:32,780 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:15:33,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:33,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:35,300 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:15:36,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:36,443 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:36,781 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:38,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:38,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:41,303 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:15:41,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:41,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:41,914 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:43,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:43,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:45,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:45,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:47,069 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:47,305 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:15:47,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:15:47,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:15:47,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:15:48,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:48,824 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:50,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:50,871 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:52,215 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:52,921 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:52,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:53,307 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:15:55,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:55,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:57,215 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:15:57,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:57,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:15:59,309 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:00,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:00,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:02,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:02,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:16:02,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:16:02,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:16:02,718 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:16:03,160 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:03,183 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:05,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:05,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:05,311 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:07,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:08,989 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:16:09,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:09,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:09,956 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:16:09,958 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:16:09,958 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:16:09,958 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:16:10,312 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:16:11,313 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:12,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:12,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:12,959 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:14,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:14,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:17,315 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:17,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:17,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:16:17,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:16:17,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:17,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:16:18,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:19,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:19,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:21,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:21,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:23,317 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:24,194 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:24,710 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:24,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:26,759 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:26,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:28,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:28,835 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:29,318 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:29,343 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:31,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:32,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:32,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:16:32,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:16:32,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:16:32,795 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:16:34,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:34,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:34,494 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:36,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:36,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:37,321 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:39,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:39,291 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:39,494 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:41,322 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:41,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:43,323 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:43,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:43,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:44,660 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:47,019 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:16:47,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:47,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:47,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:16:47,566 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:16:47,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:16:49,325 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:49,755 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:49,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:49,868 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:51,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:51,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:53,921 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:53,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:54,953 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:16:55,329 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:16:56,972 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:56,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:59,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:16:59,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:00,117 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:01,330 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:02,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:02,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:02,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:17:02,567 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:17:02,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:17:02,744 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:17:04,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:04,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:05,253 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:07,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:07,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:07,333 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:09,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:09,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:10,253 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:11,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:11,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:13,334 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:14,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:14,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:15,404 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:16,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:16,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:17,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:17:17,567 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:17:17,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:17:18,421 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:18,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:19,336 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:20,791 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:21,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:21,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:25,155 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:17:25,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:25,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:25,338 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:26,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:27,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:27,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:30,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:30,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:31,340 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:31,838 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:32,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:32,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:32,566 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:17:32,567 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:17:32,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:17:32,726 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:17:35,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:35,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:37,281 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:37,343 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:37,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:37,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:39,344 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:39,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:39,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:42,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:42,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:42,834 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:43,345 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:44,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:44,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:45,346 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:46,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:46,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:47,348 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:47,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:17:47,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:17:47,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:17:48,777 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:49,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:49,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:51,611 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:51,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:53,351 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:53,972 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:54,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:54,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:56,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:56,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:59,275 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:17:59,353 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:17:59,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:17:59,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:02,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:18:02,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:18:02,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:18:02,764 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:18:03,374 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:18:03,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:03,421 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:04,404 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:05,354 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:05,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:05,462 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:08,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:08,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:09,405 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:10,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:10,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:11,356 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:12,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:12,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:14,555 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:15,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:15,884 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:17,359 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:17,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:18:17,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:18:17,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:18:17,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:17,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:19,795 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:19,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:19,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:23,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:23,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:23,360 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:24,841 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:25,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:25,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:27,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:27,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:28,054 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:18:28,056 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:18:28,056 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:18:28,056 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:18:28,362 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:18:29,362 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:30,057 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:30,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:30,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:32,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:32,465 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:32,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:18:32,567 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:18:32,608 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:18:32,775 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:18:35,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:35,364 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:35,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:35,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:37,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:37,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:40,127 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:41,348 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:18:41,366 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:41,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:41,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:44,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:44,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:45,268 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:46,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:46,467 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:47,368 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:47,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:18:47,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:18:47,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:18:48,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:48,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:50,800 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:51,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:51,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:53,370 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:18:53,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:53,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:55,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:55,759 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:56,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:18:58,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:18:58,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:00,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:00,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:01,372 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:01,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:02,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:19:02,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:19:02,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:19:02,782 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:19:02,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:02,975 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:06,041 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:06,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:06,879 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:07,374 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:08,085 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:08,100 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:11,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:11,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:12,023 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:13,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:13,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:13,376 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:15,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:15,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:17,024 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:17,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:19:17,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:19:17,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:19:18,822 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:19:18,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:18,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:19,378 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:21,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:21,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:22,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:23,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:23,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:25,380 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:25,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:25,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:28,343 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:28,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:29,008 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:31,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:31,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:31,382 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:32,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:19:32,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:19:32,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:19:32,763 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:19:33,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:33,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:33,764 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:36,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:36,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:37,384 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:38,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:38,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:39,636 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:40,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:40,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:43,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:43,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:43,385 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:44,781 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:45,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:45,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:47,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:47,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:47,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:19:47,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:19:47,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:19:49,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:49,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:49,387 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:49,922 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:52,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:52,369 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:55,081 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:19:55,389 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:19:56,108 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:19:56,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:56,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:58,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:19:58,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:00,082 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:01,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:01,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:01,391 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:02,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:20:02,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:20:02,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:20:02,699 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:20:03,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:03,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:05,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:05,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:05,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:07,393 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:08,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:08,579 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:10,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:10,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:11,370 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:12,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:12,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:13,395 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:15,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:15,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:16,529 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:17,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:20:17,568 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:20:17,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:20:17,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:17,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:19,397 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:20,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:20,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:21,669 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:22,877 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:22,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:24,918 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:24,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:26,832 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:27,399 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:27,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:28,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:30,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:30,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:31,972 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:32,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:20:32,569 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:20:32,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:20:32,781 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:20:33,401 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:34,187 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:20:34,208 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:34,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:37,103 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:37,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:37,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:39,403 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:39,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:39,477 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:41,496 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:41,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:42,104 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:44,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:44,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:45,404 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:46,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:46,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:47,257 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:47,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:20:47,569 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:20:47,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:20:48,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:48,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:51,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:51,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:52,912 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:53,407 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:53,777 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:53,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:56,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:56,816 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:57,408 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:20:58,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:20:58,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:20:58,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:00,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:00,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:01,818 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:21:01,820 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:21:01,820 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:21:01,820 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:21:02,410 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:21:02,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:21:02,569 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:21:02,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:21:02,784 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:21:02,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:02,968 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:03,410 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:03,785 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:05,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:05,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:08,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:08,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:08,890 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:09,412 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:11,819 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:21:11,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:11,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:13,891 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:14,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:15,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:15,414 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:17,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:17,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:17,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:21:17,569 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:21:17,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:21:19,129 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:19,139 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:19,753 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:21,416 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:22,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:22,118 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:24,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:24,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:25,203 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:26,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:26,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:27,418 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:29,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:29,187 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:30,345 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:31,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:31,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:32,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:21:32,569 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:21:32,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:21:32,767 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:21:33,420 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:34,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:34,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:35,769 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:36,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:36,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:38,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:38,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:39,423 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:41,625 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:41,642 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:41,648 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:43,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:43,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:46,788 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:46,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:46,821 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:47,427 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:47,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:21:47,569 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:21:47,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:21:50,512 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:21:50,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:50,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:51,940 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:53,429 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:21:53,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:53,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:55,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:55,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:56,941 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:21:58,835 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:58,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:21:59,431 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:00,886 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:00,905 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:02,076 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:02,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:22:02,570 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:22:02,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:22:02,719 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:22:04,008 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:04,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:05,433 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:06,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:06,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:07,210 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:08,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:08,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:11,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:11,076 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:11,435 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:12,354 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:13,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:13,120 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:15,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:15,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:17,437 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:17,498 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:17,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:22:17,569 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:22:17,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:22:18,077 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:18,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:20,130 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:20,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:22,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:22,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:22,638 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:23,438 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:24,972 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:24,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:27,639 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:28,786 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:22:28,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:28,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:29,441 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:31,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:31,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:32,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:22:32,573 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:22:32,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:22:32,735 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:22:32,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:33,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:33,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:35,447 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:36,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:36,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:37,913 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:38,903 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:38,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:40,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:40,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:41,449 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:43,017 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:43,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:44,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:46,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:46,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:47,450 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:47,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:22:47,570 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:22:47,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:22:48,085 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:48,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:48,788 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:51,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:51,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:53,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:53,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:53,452 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:22:54,272 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:55,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:55,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:58,275 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:58,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:22:59,418 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:22:59,454 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:00,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:00,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:02,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:02,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:02,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:23:02,570 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:23:02,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:23:02,762 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:23:04,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:05,456 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:06,081 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:23:06,123 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:06,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:09,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:09,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:09,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:11,153 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:11,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:11,458 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:13,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:13,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:14,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:16,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:16,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:17,460 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:17,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:23:17,570 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:23:17,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:23:18,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:18,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:19,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:21,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:21,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:23,461 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:23,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:23,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:24,986 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:25,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:25,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:28,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:28,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:29,463 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:30,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:30,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:30,692 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:32,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:23:32,570 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:23:32,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:23:32,709 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:23:32,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:32,755 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:34,355 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:23:34,356 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:23:34,357 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:23:34,357 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:23:34,465 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:23:35,358 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:35,465 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:35,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:35,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:37,871 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:37,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:39,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:39,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:40,359 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:41,467 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:43,699 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:23:43,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:43,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:45,439 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:46,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:46,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:47,469 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:47,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:23:47,570 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:23:47,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:23:48,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:48,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:50,777 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:50,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:50,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:53,471 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:23:53,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:53,935 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:55,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:55,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:56,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:23:58,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:59,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:23:59,473 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:01,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:01,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:01,892 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:02,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:24:02,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:24:02,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:24:02,770 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:24:03,074 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:03,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:05,475 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:06,153 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:06,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:06,973 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:08,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:08,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:09,476 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:10,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:10,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:11,477 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:12,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:12,514 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:13,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:13,477 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:15,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:15,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:17,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:17,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:17,479 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:17,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:24:17,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:17,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:24:17,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:24:21,129 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:24:21,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:21,167 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:23,194 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:23,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:23,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:23,481 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:26,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:26,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:28,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:28,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:28,351 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:29,483 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:30,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:30,325 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:32,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:24:32,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:24:32,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:24:32,717 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:24:33,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:33,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:33,510 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:35,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:35,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:35,485 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:38,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:38,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:38,511 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:40,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:40,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:41,487 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:42,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:42,496 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:43,665 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:44,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:44,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:47,489 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:47,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:24:47,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:24:47,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:47,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:24:47,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:49,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:49,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:51,222 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:51,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:51,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:53,491 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:24:53,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:53,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:56,963 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:24:57,323 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:24:57,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:57,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:24:59,493 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:00,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:00,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:02,108 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:02,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:02,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:02,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:25:02,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:25:02,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:25:02,786 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:25:04,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:04,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:05,495 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:06,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:06,443 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:07,273 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:09,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:09,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:11,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:11,428 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:11,497 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:12,428 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:14,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:14,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:16,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:16,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:17,499 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:17,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:25:17,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:17,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:25:17,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:25:18,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:18,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:21,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:21,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:22,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:23,501 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:23,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:23,710 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:25,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:25,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:27,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:28,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:28,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:29,503 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:30,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:30,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:32,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:25:32,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:25:32,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:25:32,797 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:25:32,798 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:34,677 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:25:34,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:34,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:35,505 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:36,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:36,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:38,038 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:39,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:39,757 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:41,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:41,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:43,195 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:43,508 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:43,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:43,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:45,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:45,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:47,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:25:47,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:25:47,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:25:48,653 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:48,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:48,794 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:49,510 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:50,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:50,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:52,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:52,745 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:53,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:25:53,579 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:25:53,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:25:53,580 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:25:54,512 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:25:54,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:55,512 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:25:55,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:55,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:57,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:57,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:59,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:25:59,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:25:59,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:01,516 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:02,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:26:02,572 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:26:02,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:26:02,768 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:26:02,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:02,886 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:04,769 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:04,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:04,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:06,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:06,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:07,518 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:09,804 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:10,654 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:26:10,676 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:10,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:13,520 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:13,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:13,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:14,960 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:15,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:15,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:17,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:26:17,571 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:26:17,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:26:17,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:18,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:19,521 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:20,120 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:20,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:20,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:22,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:22,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:25,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:25,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:25,265 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:25,523 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:27,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:27,836 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:29,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:29,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:30,266 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:31,525 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:31,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:31,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:32,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:26:32,572 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:26:32,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:26:32,729 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:26:35,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:35,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:35,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:37,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:37,116 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:37,527 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:40,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:40,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:41,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:42,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:42,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:43,531 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:45,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:45,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:46,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:47,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:26:47,572 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:26:47,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:26:49,174 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:26:49,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:49,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:51,534 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:52,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:52,310 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:52,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:54,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:54,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:55,535 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:26:57,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:57,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:58,011 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:26:59,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:26:59,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:02,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:27:02,572 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:27:02,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:27:02,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:02,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:02,712 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:27:03,538 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:03,713 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:04,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:04,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:06,770 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:06,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:09,300 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:09,539 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:09,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:09,717 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:11,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:11,773 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:13,816 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:13,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:14,435 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:15,541 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:16,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:16,903 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:17,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:27:17,572 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:27:17,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:27:18,947 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:18,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:19,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:20,993 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:21,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:21,543 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:24,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:24,063 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:24,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:27,546 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:27,691 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:27:27,710 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:27,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:29,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:29,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:29,757 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:32,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:27:32,572 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:27:32,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:27:32,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:32,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:32,779 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:27:33,548 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:34,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:34,745 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:34,780 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:37,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:37,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:39,550 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:39,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:39,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:40,029 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:42,893 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:42,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:44,937 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:44,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:45,168 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:45,552 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:46,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:47,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:47,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:27:47,572 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:27:47,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:27:50,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:50,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:50,477 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:51,554 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:52,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:52,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:54,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:54,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:55,478 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:27:57,189 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:57,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:57,556 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:27:59,232 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:27:59,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:01,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:01,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:01,465 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:02,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:28:02,573 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:28:02,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:28:02,746 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:28:03,558 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:04,939 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:28:04,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:04,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:06,747 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:08,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:08,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:09,559 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:10,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:10,258 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:11,751 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:12,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:12,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:15,364 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:15,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:15,562 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:17,015 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:17,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:17,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:17,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:28:17,573 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:28:17,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:28:20,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:20,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:21,563 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:22,154 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:22,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:22,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:24,580 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:24,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:27,155 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:27,377 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:28:27,379 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:28:27,379 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:28:27,380 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:28:27,565 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:28:27,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:27,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:29,566 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:29,645 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:29,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:31,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:31,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:32,381 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:32,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:28:32,573 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:28:32,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:28:32,800 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:28:34,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:34,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:35,568 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:36,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:36,936 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:37,442 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:38,960 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:38,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:41,570 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:42,563 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:28:42,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:42,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:42,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:45,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:45,825 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:47,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:28:47,573 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:47,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:28:47,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:28:47,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:47,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:47,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:49,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:49,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:52,877 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:52,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:53,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:53,575 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:28:55,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:55,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:57,878 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:28:58,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:58,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:28:59,577 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:00,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:00,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:02,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:02,153 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:02,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:29:02,573 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:29:02,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:29:02,793 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:29:03,794 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:05,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:05,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:05,579 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:07,307 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:07,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:09,178 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:09,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:09,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:11,581 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:12,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:12,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:14,317 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:14,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:14,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:16,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:16,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:17,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:29:17,573 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:29:17,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:29:17,583 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:20,402 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:29:20,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:20,440 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:20,609 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:23,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:23,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:23,585 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:25,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:25,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:25,611 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:27,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:27,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:29,587 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:30,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:30,588 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:30,756 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:32,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:29:32,573 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:29:32,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:29:32,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:32,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:32,716 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:29:34,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:34,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:35,589 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:35,923 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:37,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:37,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:39,581 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:39,591 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:40,923 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:41,591 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:41,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:41,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:44,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:44,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:46,053 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:46,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:46,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:47,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:29:47,573 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:29:47,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:29:47,593 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:48,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:48,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:50,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:50,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:51,756 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:53,595 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:29:53,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:53,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:57,369 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:29:57,687 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:29:57,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:57,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:59,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:29:59,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:01,598 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:01,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:01,792 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:02,521 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:02,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:30:02,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:30:02,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:30:02,799 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:30:04,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:04,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:06,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:06,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:07,599 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:07,668 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:08,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:08,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:12,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:12,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:12,832 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:13,601 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:14,129 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:14,139 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:17,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:17,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:17,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:30:17,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:30:17,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:30:17,983 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:19,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:19,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:19,603 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:21,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:21,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:22,984 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:24,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:24,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:25,605 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:26,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:26,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:28,222 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:28,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:28,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:31,607 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:31,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:31,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:32,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:30:32,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:30:32,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:30:32,779 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:30:33,240 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:33,608 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:35,458 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:30:35,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:35,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:37,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:37,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:37,609 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:38,777 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:39,610 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:40,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:40,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:42,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:42,971 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:43,611 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:44,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:44,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:45,024 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:46,212 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:30:46,213 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:30:46,213 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:30:46,214 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:30:46,612 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:30:47,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:30:47,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:30:47,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:30:47,613 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:48,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:48,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:48,614 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:49,614 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:49,971 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:50,080 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:50,090 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:52,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:52,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:53,615 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:55,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:55,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:55,517 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:30:55,616 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:30:57,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:57,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:59,110 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:59,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:30:59,617 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:01,479 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:02,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:02,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:02,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:31:02,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:31:02,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:31:02,774 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:31:03,618 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:04,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:04,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:06,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:06,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:07,440 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:09,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:09,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:09,620 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:12,978 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:13,004 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:31:13,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:13,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:15,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:15,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:15,623 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:17,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:17,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:17,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:31:17,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:31:17,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:31:18,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:20,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:20,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:21,625 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:22,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:22,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:23,962 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:24,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:24,187 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:27,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:27,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:28,627 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:29,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:29,332 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:29,491 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:31,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:31,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:32,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:31:32,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:31:32,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:31:32,789 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:31:34,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:34,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:34,629 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:34,790 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:36,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:36,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:39,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:39,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:39,998 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:40,630 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:41,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:41,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:44,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:44,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:45,953 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:46,632 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:46,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:46,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:47,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:31:47,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:31:47,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:31:50,267 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:31:50,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:50,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:50,634 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:51,511 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:53,130 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:53,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:55,160 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:55,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:56,636 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:31:57,509 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:31:58,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:31:58,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:00,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:00,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:02,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:32:02,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:02,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:32:02,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:32:02,640 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:02,780 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:32:03,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:03,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:05,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:05,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:07,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:07,428 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:07,916 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:08,642 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:10,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:10,430 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:12,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:12,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:13,873 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:14,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:14,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:14,645 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:17,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:17,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:17,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:32:17,574 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:32:17,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:32:19,412 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:19,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:19,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:20,657 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:22,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:22,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:24,588 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:24,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:25,363 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:26,652 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:28,308 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:32:28,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:28,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:30,821 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:31,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:31,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:32,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:32:32,575 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:32:32,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:32:32,654 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:32,775 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:32:33,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:33,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:35,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:35,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:36,777 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:38,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:38,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:38,657 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:40,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:40,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:42,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:42,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:42,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:44,660 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:45,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:45,239 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:47,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:47,275 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:47,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:32:47,575 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:32:47,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:32:47,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:48,661 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:50,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:50,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:52,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:52,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:52,760 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:54,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:54,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:54,663 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:32:57,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:57,322 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:58,254 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:32:58,845 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:32:58,847 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:32:58,847 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:32:58,848 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:32:59,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:59,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:32:59,665 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:33:00,666 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:01,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:01,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:02,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:33:02,576 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:33:02,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:33:02,778 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:33:03,779 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:04,875 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:33:04,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:04,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:06,668 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:07,960 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:07,971 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:09,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:09,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:10,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:12,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:12,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:12,671 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:15,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:15,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:15,289 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:17,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:17,148 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:17,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:33:17,576 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:33:17,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:33:18,673 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:19,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:19,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:21,382 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:22,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:22,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:24,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:24,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:24,675 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:26,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:26,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:26,976 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:29,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:29,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:30,677 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:31,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:31,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:32,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:32,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:33:32,576 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:33:32,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:33:32,782 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:33:33,207 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:33,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:34,678 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:36,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:36,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:37,608 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:38,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:38,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:40,681 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:41,975 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:33:42,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:42,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:43,158 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:45,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:45,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:46,683 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:47,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:47,209 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:47,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:33:47,576 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:33:47,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:33:48,713 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:49,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:49,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:52,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:52,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:52,685 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:54,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:54,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:54,477 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:33:56,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:56,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:58,687 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:33:59,239 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:33:59,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:00,283 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:01,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:01,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:02,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:34:02,576 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:34:02,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:34:02,802 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:34:03,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:03,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:04,689 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:05,803 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:06,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:06,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:08,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:08,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:10,691 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:10,893 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:11,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:11,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:13,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:13,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:15,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:15,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:16,694 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:16,851 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:17,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:34:17,576 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:34:17,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:34:18,980 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:34:19,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:19,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:20,695 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:21,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:21,905 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:22,091 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:23,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:23,935 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:25,955 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:25,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:26,697 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:28,073 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:28,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:28,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:30,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:31,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:32,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:34:32,576 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:34:32,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:34:32,699 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:32,738 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:34:33,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:33,035 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:33,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:36,080 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:36,090 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:38,110 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:38,120 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:38,701 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:38,779 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:40,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:40,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:43,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:43,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:44,609 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:44,703 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:45,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:45,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:47,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:47,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:47,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:34:47,577 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:34:47,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:34:49,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:50,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:50,457 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:50,705 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:52,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:52,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:55,804 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:34:55,825 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:55,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:56,417 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:34:56,707 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:34:58,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:34:58,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:00,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:01,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:01,970 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:02,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:35:02,577 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:35:02,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:35:02,709 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:02,782 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:35:03,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:03,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:06,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:06,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:07,931 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:08,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:08,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:08,711 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:10,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:10,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:12,713 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:13,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:13,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:13,488 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:15,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:15,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:17,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:17,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:17,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:35:17,577 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:35:17,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:35:18,715 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:18,794 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:20,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:20,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:22,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:22,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:23,050 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:35:23,051 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:35:23,052 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:35:23,052 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:35:23,716 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:35:24,053 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:24,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:24,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:24,717 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:27,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:27,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:29,054 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:29,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:29,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:30,718 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:32,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:35:32,577 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:35:32,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:35:32,742 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:35:33,319 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:35:33,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:33,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:34,744 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:36,298 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:36,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:36,720 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:38,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:38,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:40,089 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:40,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:40,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:42,722 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:43,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:43,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:45,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:45,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:45,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:47,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:47,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:47,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:35:47,577 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:35:47,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:35:48,724 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:50,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:50,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:50,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:52,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:52,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:54,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:54,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:54,726 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:35:56,229 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:35:57,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:57,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:59,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:35:59,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:00,729 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:01,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:01,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:02,188 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:02,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:36:02,578 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:36:02,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:36:02,752 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:36:03,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:04,730 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:04,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:06,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:06,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:07,797 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:10,563 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:36:10,581 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:10,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:10,732 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:13,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:13,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:13,760 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:15,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:15,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:16,735 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:17,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:36:17,578 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:36:17,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:36:17,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:17,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:18,774 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:20,884 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:20,894 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:22,737 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:22,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:22,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:24,301 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:24,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:24,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:28,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:28,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:28,739 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:30,052 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:30,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:30,250 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:32,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:32,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:32,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:36:32,578 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:36:32,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:36:32,744 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:36:34,741 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:35,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:35,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:35,820 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:37,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:37,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:39,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:39,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:40,743 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:41,193 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:42,322 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:42,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:44,366 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:44,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:46,193 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:47,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:36:47,579 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:36:47,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:36:48,182 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:36:48,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:48,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:51,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:51,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:51,404 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:53,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:53,260 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:56,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:56,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:56,747 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:36:57,031 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:36:58,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:36:58,424 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:01,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:01,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:02,196 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:02,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:37:02,578 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:37:02,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:37:02,750 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:02,771 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:37:03,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:03,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:06,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:06,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:07,814 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:08,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:08,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:08,752 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:10,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:10,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:13,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:13,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:13,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:14,754 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:15,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:15,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:17,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:37:17,579 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:37:17,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:37:18,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:18,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:19,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:20,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:20,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:20,756 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:23,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:23,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:25,245 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:26,758 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:27,380 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:37:27,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:27,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:29,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:29,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:31,190 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:32,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:32,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:32,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:37:32,579 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:37:32,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:37:32,762 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:32,832 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:37:34,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:34,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:36,477 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:36,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:36,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:36,764 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:39,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:39,366 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:41,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:41,411 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:42,659 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:42,766 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:43,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:43,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:46,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:46,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:47,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:37:47,579 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:37:47,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:37:47,798 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:48,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:48,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:48,768 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:51,430 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:51,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:52,798 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:53,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:53,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:54,773 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:37:56,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:56,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:58,478 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:37:58,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:58,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:37:59,327 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:37:59,332 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:37:59,336 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:37:59,338 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:37:59,777 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:38:00,778 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:01,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:01,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:02,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:38:02,579 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:38:02,620 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:38:02,774 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:38:03,775 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:05,091 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:38:05,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:05,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:06,779 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:07,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:07,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:09,170 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:09,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:10,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:12,781 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:13,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:13,209 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:15,115 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:15,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:15,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:17,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:17,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:17,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:38:17,579 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:38:17,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:38:18,783 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:20,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:20,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:21,174 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:22,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:22,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:24,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:24,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:24,785 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:26,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:27,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:27,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:29,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:29,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:30,787 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:31,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:31,860 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:32,133 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:32,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:38:32,579 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:38:32,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:38:32,752 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:38:34,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:34,907 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:36,790 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:36,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:36,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:38,082 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:38,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:39,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:40,792 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:42,484 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:38:42,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:42,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:43,641 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:45,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:45,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:46,794 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:47,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:38:47,579 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:38:47,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:38:47,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:47,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:48,769 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:50,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:50,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:52,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:52,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:52,795 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:54,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:38:54,777 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:54,790 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:57,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:57,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:58,797 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:38:59,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:38:59,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:00,135 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:02,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:02,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:02,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:39:02,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:39:02,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:39:02,754 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:39:04,800 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:05,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:05,079 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:06,085 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:07,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:07,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:09,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:09,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:10,802 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:11,673 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:12,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:12,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:14,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:14,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:16,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:16,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:16,805 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:17,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:17,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:39:17,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:39:17,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:39:22,434 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:39:22,496 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:22,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:22,806 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:23,169 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:24,847 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:24,868 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:26,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:26,903 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:28,808 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:28,926 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:28,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:29,132 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:31,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:31,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:32,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:39:32,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:39:32,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:39:32,768 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:39:32,810 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:34,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:34,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:34,769 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:37,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:37,187 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:38,812 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:39,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:39,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:40,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:41,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:41,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:44,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:44,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:44,814 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:46,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:46,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:46,654 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:47,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:39:47,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:39:47,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:39:48,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:48,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:50,816 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:51,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:51,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:52,195 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:53,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:53,773 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:55,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:55,824 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:56,818 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:39:58,159 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:39:59,413 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:39:59,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:39:59,458 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:02,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:02,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:02,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:40:02,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:40:02,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:40:02,757 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:40:02,820 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:03,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:04,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:04,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:06,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:06,633 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:08,822 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:09,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:09,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:09,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:11,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:11,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:13,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:13,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:14,824 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:15,543 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:16,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:16,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:17,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:40:17,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:40:17,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:40:18,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:18,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:20,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:20,759 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:21,017 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:22,827 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:23,835 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:23,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:24,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:40:24,533 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:40:24,534 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:40:24,534 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:40:24,827 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:40:25,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:25,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:26,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:26,828 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:27,937 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:27,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:30,830 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:31,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:31,059 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:32,080 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:32,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:40:32,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:40:32,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:40:32,758 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:40:33,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:33,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:36,893 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:40:36,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:36,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:37,200 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:38,832 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:40,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:40,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:42,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:42,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:42,834 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:43,042 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:44,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:44,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:47,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:47,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:47,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:40:47,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:40:47,766 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:40:48,836 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:49,003 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:49,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:49,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:51,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:51,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:54,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:54,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:54,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:54,838 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:40:56,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:56,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:59,579 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:40:59,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:40:59,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:00,840 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:01,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:01,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:02,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:41:02,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:41:02,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:41:02,784 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:41:03,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:04,786 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:04,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:06,842 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:07,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:07,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:09,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:09,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:10,024 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:12,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:12,138 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:12,844 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:15,934 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:41:15,953 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:15,977 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:15,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:17,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:41:17,580 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:41:17,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:41:18,040 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:18,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:18,845 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:20,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:20,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:21,812 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:23,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:23,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:24,847 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:25,100 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:25,112 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:27,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:27,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:27,576 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:30,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:30,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:30,849 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:32,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:32,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:32,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:32,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:41:32,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:41:32,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:41:32,775 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:41:34,232 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:34,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:34,851 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:37,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:37,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:38,220 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:39,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:39,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:42,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:42,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:42,853 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:43,319 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:44,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:44,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:46,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:46,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:46,854 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:47,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:41:47,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:41:47,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:41:48,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:49,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:49,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:52,857 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:53,358 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:41:53,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:53,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:53,922 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:41:55,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:55,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:58,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:58,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:41:58,859 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:41:59,881 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:00,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:00,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:02,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:02,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:42:02,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:42:02,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:02,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:42:02,770 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:42:04,861 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:05,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:05,510 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:05,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:07,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:07,569 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:09,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:09,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:10,863 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:11,467 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:12,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:12,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:14,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:14,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:16,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:16,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:16,865 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:17,006 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:17,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:42:17,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:42:17,621 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:42:19,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:19,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:21,968 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:21,977 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:22,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:22,867 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:24,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:24,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:26,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:26,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:28,448 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:28,869 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:30,594 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:42:30,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:31,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:32,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:42:32,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:42:32,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:42:32,765 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:42:33,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:33,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:33,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:34,871 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:35,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:35,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:38,579 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:38,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:39,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:40,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:40,649 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:40,873 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:43,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:43,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:45,546 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:45,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:45,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:46,876 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:47,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:42:47,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:42:47,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:42:48,658 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:48,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:50,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:42:50,706 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:50,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:51,210 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:42:51,210 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:42:51,211 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:51,211 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:42:51,878 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:42:52,878 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:53,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:53,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:55,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:55,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:56,212 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:42:56,880 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:42:58,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:42:58,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:00,905 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:00,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:01,303 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:02,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:43:02,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:43:02,624 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:43:02,767 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:43:03,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:03,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:04,882 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:05,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:05,986 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:06,768 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:08,884 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:09,706 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:43:10,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:10,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:12,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:13,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:14,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:14,886 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:16,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:16,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:17,203 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:17,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:43:17,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:43:17,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:43:19,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:19,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:21,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:21,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:22,612 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:22,888 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:24,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:24,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:26,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:26,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:26,889 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:27,977 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:29,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:29,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:31,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:31,625 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:32,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:43:32,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:43:32,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:43:32,892 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:32,927 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:43:33,929 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:34,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:34,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:36,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:36,688 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:38,894 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:39,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:39,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:39,677 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:41,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:41,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:44,770 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:44,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:44,896 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:45,123 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:47,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:47,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:43:47,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:43:47,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:43:48,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:50,782 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:51,898 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:52,326 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:43:52,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:52,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:55,579 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:55,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:56,334 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:43:57,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:57,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:57,900 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:43:59,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:43:59,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:01,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:01,902 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:02,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:44:02,582 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:44:02,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:44:02,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:02,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:02,775 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:44:04,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:04,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:06,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:06,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:07,843 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:07,904 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:09,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:09,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:11,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:11,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:13,632 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:13,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:13,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:13,906 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:17,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:17,127 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:17,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:44:17,582 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:44:17,625 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:44:19,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:19,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:19,397 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:19,908 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:21,183 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:21,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:24,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:24,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:25,350 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:25,910 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:26,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:26,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:29,707 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:44:29,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:29,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:30,949 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:31,913 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:32,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:44:32,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:44:32,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:44:32,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:32,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:32,813 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:44:34,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:34,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:36,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:36,885 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:37,915 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:37,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:39,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:39,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:41,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:42,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:42,431 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:43,917 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:45,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:45,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:47,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:47,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:47,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:47,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:44:47,581 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:44:47,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:44:49,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:49,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:49,920 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:52,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:52,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:52,924 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:53,922 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:44:54,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:54,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:56,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:56,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:57,925 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:44:59,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:59,580 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:44:59,924 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:01,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:01,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:02,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:45:02,582 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:45:02,624 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:45:02,774 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:45:03,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:03,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:03,775 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:05,926 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:07,204 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:45:07,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:07,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:08,821 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:10,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:10,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:11,927 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:12,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:12,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:14,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:14,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:14,443 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:16,463 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:45:16,465 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:45:16,465 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:45:16,465 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:45:16,929 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:45:17,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:17,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:17,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:45:17,582 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:45:17,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:45:17,929 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:18,930 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:19,307 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:19,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:19,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:21,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:21,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:23,933 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:24,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:24,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:25,022 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:26,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:26,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:28,401 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:28,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:29,934 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:30,974 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:31,476 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:31,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:32,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:45:32,582 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:45:32,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:45:32,792 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:45:33,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:33,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:35,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:35,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:35,936 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:36,573 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:38,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:38,633 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:40,653 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:40,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:41,938 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:42,374 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:44,502 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:45:44,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:44,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:47,494 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:47,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:45:47,582 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:45:47,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:45:47,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:47,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:47,940 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:49,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:49,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:51,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:51,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:53,463 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:53,942 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:45:54,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:54,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:56,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:56,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:58,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:58,649 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:45:59,416 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:45:59,944 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:01,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:01,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:02,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:46:02,583 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:46:02,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:46:02,778 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:46:03,944 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:03,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:03,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:04,931 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:06,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:06,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:09,021 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:09,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:09,945 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:10,888 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:11,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:11,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:14,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:14,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:15,947 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:16,221 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:16,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:16,282 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:17,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:46:17,582 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:46:17,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:46:18,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:18,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:21,951 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:22,234 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:22,412 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:46:23,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:23,017 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:25,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:25,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:27,077 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:27,090 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:27,805 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:27,953 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:30,332 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:30,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:32,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:32,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:32,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:46:32,583 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:46:32,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:46:32,806 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:46:32,807 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:33,955 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:34,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:34,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:36,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:36,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:38,796 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:39,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:39,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:39,957 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:41,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:41,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:43,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:43,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:44,473 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:45,958 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:46,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:46,476 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:47,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:46:47,583 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:46:47,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:46:48,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:48,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:49,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:50,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:50,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:51,960 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:53,635 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:53,648 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:54,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:46:55,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:55,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:55,962 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:46:59,392 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:46:59,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:46:59,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:00,140 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:02,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:02,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:02,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:47:02,583 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:47:02,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:47:02,794 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:47:03,964 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:04,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:04,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:05,796 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:06,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:06,635 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:07,965 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:09,804 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:09,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:10,896 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:11,838 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:11,848 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:13,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:13,884 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:13,968 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:16,041 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:16,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:16,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:17,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:47:17,583 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:47:17,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:47:19,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:19,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:21,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:21,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:21,174 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:21,971 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:23,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:23,886 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:25,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:25,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:25,972 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:26,844 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:27,948 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:27,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:31,035 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:31,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:31,958 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:31,974 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:32,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:47:32,583 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:47:32,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:47:32,782 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:47:33,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:33,121 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:37,919 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:37,979 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:38,064 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:47:38,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:38,100 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:40,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:40,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:42,167 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:42,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:42,920 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:43,288 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 15:47:43,289 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 15:47:43,289 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 15:47:43,290 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 15:47:43,981 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 15:47:44,982 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:45,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:45,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:45,982 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:47,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:47,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:47,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:47:47,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:47:47,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:47:48,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:49,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:49,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:52,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:52,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:53,747 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:54,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:54,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:56,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:56,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:57,986 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:47:58,878 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:47:59,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:47:59,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:01,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:01,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:02,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:48:02,586 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:48:02,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:48:02,784 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:48:03,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:03,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:04,785 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:06,414 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:06,443 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:08,462 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:08,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:09,990 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:48:10,191 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:11,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:11,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:14,953 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:48:14,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:14,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:15,191 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:17,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:48:17,583 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:48:17,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:48:18,029 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:18,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:20,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:20,071 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:20,811 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:23,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:23,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:23,994 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:48:25,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:25,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:26,005 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:27,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:28,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:30,110 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:30,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:31,006 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:32,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:48:32,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:48:32,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:48:32,796 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:48:33,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:33,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:35,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:35,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:35,998 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:48:36,748 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:38,158 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:38,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:40,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:40,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:41,748 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:42,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:42,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:45,035 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:45,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:46,750 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:47,079 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:47,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:47,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:48:47,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:48:47,624 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:48:50,002 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:48:50,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:50,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:51,765 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:53,771 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:48:53,790 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:53,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:56,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:56,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:56,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:48:58,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:48:58,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:00,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:00,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:02,005 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:49:02,402 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:02,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:49:02,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:49:02,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:49:02,798 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:49:03,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:03,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:05,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:05,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:07,800 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:08,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:08,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:10,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:10,801 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:12,801 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:13,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:13,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:14,009 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:49:15,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:15,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:17,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:49:17,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:49:17,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:49:18,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:18,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:18,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:20,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:20,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:22,790 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:22,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:23,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:25,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:25,658 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:26,013 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:49:27,688 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:27,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:29,252 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:31,123 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:49:31,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:31,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:32,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:49:32,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:49:32,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:49:32,795 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:49:33,948 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:33,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:34,797 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:35,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:35,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:38,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:38,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:40,017 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:49:40,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:41,118 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:41,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:43,158 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:43,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:45,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:45,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:45,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:47,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:49:47,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:49:47,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:49:48,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:48,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:50,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:50,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:51,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:52,021 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:49:52,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:52,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:55,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:55,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:56,127 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:49:57,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:57,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:59,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:49:59,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:01,128 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:02,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:02,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:02,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:50:02,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:50:02,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:50:02,791 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:50:04,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:04,209 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:06,025 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:50:06,374 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:07,942 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:50:07,965 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:07,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:11,058 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:11,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:11,375 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:13,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:13,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:15,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:15,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:16,940 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:17,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:50:17,584 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:50:17,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:50:18,030 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:50:18,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:18,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:20,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:20,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:22,813 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:23,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:23,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:25,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:25,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:27,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:27,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:27,814 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:30,034 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:50:30,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:30,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:32,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:32,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:32,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:50:32,585 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:50:32,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:50:32,793 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:50:33,794 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:34,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:34,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:37,232 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:37,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:38,796 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:39,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:39,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:41,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:41,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:42,038 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:50:44,474 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:44,825 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:50:44,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:44,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:47,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:50:47,585 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:50:47,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:50:47,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:47,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:49,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:49,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:49,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:51,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:51,792 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:54,042 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:50:54,684 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:54,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:55,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:50:56,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:56,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:58,750 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:50:58,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:00,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:01,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:01,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:02,584 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:51:02,585 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:51:02,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:51:02,795 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:51:03,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:03,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:06,045 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:51:06,169 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:06,606 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:06,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:08,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:08,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:10,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:10,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:11,170 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:13,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:13,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:15,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:15,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:16,171 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:17,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:51:17,585 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:51:17,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:51:17,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:17,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:20,049 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:51:21,405 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:51:21,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:21,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:21,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:24,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:24,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:26,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:26,443 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:26,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:28,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:28,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:31,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:31,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:31,778 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:32,053 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:51:32,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:51:32,586 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:51:32,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:51:32,796 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:51:33,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:33,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:35,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:35,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:36,798 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:38,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:38,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:40,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:40,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:41,799 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:42,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:42,674 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:45,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:45,633 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:46,057 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:51:47,582 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:47,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:51:47,585 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:51:47,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:51:47,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:47,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:50,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:50,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:52,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:52,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:53,192 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:51:54,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:54,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:56,059 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:51:58,079 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:51:58,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:58,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:51:59,079 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:01,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:01,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:02,061 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:52:02,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:52:02,632 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:52:02,632 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:52:02,798 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:52:03,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:04,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:04,799 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:06,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:07,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:08,063 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:52:10,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:10,239 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:10,433 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:13,239 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:13,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:15,065 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:52:15,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:15,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:16,051 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:17,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:52:17,586 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:52:17,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:52:18,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:18,658 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:20,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:20,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:21,788 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:22,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:22,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:25,068 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:52:25,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:25,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:27,072 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:27,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:27,773 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:29,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:29,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:32,073 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:32,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:52:32,586 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:52:32,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:52:32,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:32,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:32,800 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:52:34,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:34,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:37,661 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:38,414 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:52:39,072 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:52:39,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:39,801 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:41,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:41,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:42,662 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:43,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:43,864 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:46,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:46,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:47,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:52:47,586 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:52:47,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:52:47,784 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:48,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:48,935 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:50,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:50,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:51,076 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:52:52,854 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:53,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:53,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:56,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:56,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:57,854 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:52:58,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:52:58,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:00,130 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:00,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:02,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:53:02,586 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:53:02,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:53:02,802 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:53:03,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:03,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:03,605 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:05,080 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:53:05,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:05,222 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:07,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:07,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:08,654 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:10,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:10,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:12,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:12,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:13,655 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:16,297 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:53:16,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:16,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:17,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:53:17,586 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:53:17,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:53:18,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:18,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:19,084 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:53:20,540 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:21,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:21,370 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:23,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:23,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:25,540 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:26,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:26,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:28,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:28,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:30,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:30,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:30,591 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:32,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:53:32,587 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:53:32,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:53:32,806 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:53:33,088 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:53:33,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:33,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:35,807 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:35,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:35,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:37,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:37,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:40,808 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:40,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:40,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:42,936 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:42,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:44,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:45,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:45,091 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:53:45,969 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:47,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:53:47,586 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:53:47,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:53:48,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:48,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:50,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:50,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:51,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:53,936 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:53:53,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:53,965 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:56,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:53:57,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:57,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:59,095 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:53:59,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:53:59,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:01,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:01,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:02,244 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:02,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:54:02,587 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:54:02,595 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:54:02,806 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:54:04,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:04,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:06,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:06,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:07,807 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:09,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:09,370 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:11,401 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:11,424 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:13,099 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:54:13,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:13,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:13,565 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:16,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:16,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:17,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:54:17,587 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:54:17,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:54:18,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:18,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:18,782 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:20,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:20,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:23,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:23,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:24,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:25,103 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:54:25,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:25,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:28,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:28,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:29,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:32,161 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:54:32,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:32,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:32,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:54:32,587 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:54:32,595 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:54:32,807 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:54:34,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:34,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:34,809 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:37,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:37,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:39,107 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:54:39,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:39,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:39,898 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:42,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:42,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:44,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:44,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:44,898 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:46,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:46,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:47,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:54:47,587 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:54:47,595 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:54:49,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:49,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:50,847 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:51,111 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:54:51,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:51,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:54,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:54,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:55,848 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:54:56,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:56,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:59,424 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:54:59,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:00,849 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:01,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:01,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:02,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:55:02,587 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:55:02,595 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:55:02,809 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:55:04,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:04,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:05,115 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:55:06,162 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:06,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:06,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:10,305 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:55:10,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:10,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:11,162 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:13,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:13,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:15,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:15,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:16,163 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:17,121 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:55:17,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:55:17,594 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:55:17,602 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:55:18,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:18,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:20,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:20,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:21,326 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:23,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:23,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:25,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:25,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:26,327 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:28,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:28,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:29,126 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:55:31,658 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:31,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:32,214 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:32,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:55:32,587 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:55:32,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:55:32,810 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:55:34,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:34,770 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:36,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:36,817 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:37,812 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:38,837 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:38,864 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:39,129 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:55:41,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:41,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:43,066 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:43,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:43,993 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:46,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:46,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:47,586 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:55:47,587 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:55:47,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:55:48,812 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:49,584 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:55:49,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:49,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:52,625 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:52,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:53,133 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:55:54,193 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:54,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:54,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:56,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:56,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:59,194 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:55:59,827 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:55:59,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:01,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:01,934 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:02,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:56:02,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:56:02,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:56:02,819 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:56:03,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:03,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:04,225 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:05,137 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:56:06,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:06,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:08,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:08,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:09,270 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:11,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:11,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:13,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:13,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:14,271 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:16,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:16,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:17,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:56:17,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:56:17,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:56:18,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:18,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:19,141 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:56:19,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:20,539 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:20,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:23,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:23,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:24,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:27,253 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:56:27,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:27,293 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:29,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:29,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:29,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:32,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:32,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:32,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:56:32,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:56:32,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:56:32,813 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:56:33,145 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:56:34,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:34,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:34,815 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:36,602 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:37,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:39,768 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:39,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:39,815 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:41,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:41,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:44,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:44,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:45,149 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:56:45,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:46,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:46,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:47,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:56:47,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:56:47,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:56:50,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:50,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:50,802 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:52,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:52,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:55,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:55,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:55,803 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:56:57,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:57,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:59,153 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:56:59,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:56:59,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:01,748 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:02,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:02,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:02,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:57:02,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:57:02,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:57:02,815 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:57:05,960 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:57:05,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:05,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:06,816 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:09,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:09,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:11,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:11,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:11,157 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:57:11,890 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:13,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:13,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:16,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:16,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:16,890 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:17,587 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:57:17,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:57:17,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:57:18,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:18,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:21,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:21,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:22,782 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:23,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:23,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:25,161 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:57:25,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:25,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:28,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:28,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:28,088 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:31,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:31,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:32,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:57:32,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:57:32,596 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:57:32,817 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:57:33,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:33,322 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:33,818 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:35,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:35,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:38,374 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:38,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:39,166 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:57:39,630 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:40,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:40,411 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:44,104 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:57:44,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:44,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:44,630 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:47,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:47,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:47,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:57:47,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:57:47,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:57:49,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:49,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:49,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:51,169 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:57:52,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:52,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:54,325 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:54,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:54,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:57:57,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:57,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:59,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:59,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:57:59,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:01,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:01,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:02,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:58:02,589 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:58:02,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:58:02,819 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:58:04,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:04,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:05,049 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:05,173 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:58:06,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:06,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:08,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:08,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:10,050 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:12,076 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:12,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:13,176 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:58:14,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:14,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:15,308 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:16,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:17,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:17,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:58:17,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:58:17,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:58:19,177 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:58:19,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:19,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:20,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:23,296 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:58:23,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:23,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:26,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:26,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:26,530 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:27,180 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:58:28,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:28,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:30,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:30,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:31,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:32,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:58:32,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:58:32,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:58:32,823 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:58:33,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:33,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:35,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:35,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:36,824 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:38,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:38,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:40,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:40,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:41,184 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:58:42,790 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:42,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:42,835 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:44,864 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:44,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:47,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:58:47,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:58:47,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:58:47,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:47,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:49,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:50,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:50,595 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:52,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:52,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:53,188 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:58:55,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:55,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:55,855 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:58:57,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:58:57,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:00,856 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:01,041 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:59:01,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:01,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:02,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:59:02,589 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:59:02,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:59:02,822 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:59:04,071 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:04,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:05,981 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:06,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:06,127 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:07,192 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:59:08,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:08,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:10,982 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:11,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:11,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:13,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:13,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:15,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:15,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:15,982 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:17,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:59:17,589 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:59:17,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:59:18,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:18,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:19,196 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:59:20,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:20,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:21,142 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:22,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:22,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:25,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:25,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:26,143 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:27,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:27,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:29,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:29,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:31,143 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:32,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:59:32,588 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:59:32,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:59:32,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:32,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:32,824 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 15:59:33,200 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:59:34,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:34,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:36,825 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:38,417 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 15:59:38,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:38,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:41,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:41,645 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:41,826 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:43,692 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:43,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:45,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:45,779 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:47,204 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 15:59:47,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 15:59:47,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:47,589 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 15:59:47,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 15:59:48,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:48,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:50,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:50,868 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:52,920 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:52,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:53,435 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:56,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:56,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:58,121 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:58,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 15:59:58,793 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 15:59:59,208 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:00:00,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:00,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:02,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:00:02,589 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:00:02,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:00:02,826 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:00:03,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:03,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:03,827 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:05,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:05,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:07,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:07,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:08,827 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:10,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:10,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:12,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:12,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:13,212 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:00:14,061 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:16,113 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:00:16,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:16,158 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:17,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:00:17,589 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:00:17,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:00:19,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:19,065 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:19,785 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:21,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:21,099 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:23,121 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:23,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:25,068 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:25,216 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:00:26,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:26,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:28,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:28,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:30,069 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:30,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:31,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:32,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:00:32,589 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:00:32,598 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:00:32,828 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:00:33,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:33,043 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:35,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:35,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:35,829 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:37,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:38,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:39,220 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:00:40,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:40,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:41,190 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:43,024 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:43,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:45,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:45,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:46,190 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:47,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:00:47,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:00:47,598 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:00:48,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:48,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:50,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:50,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:51,224 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:00:51,336 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:53,630 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:00:53,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:53,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:56,337 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:00:56,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:56,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:58,467 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:00:58,476 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:00,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:00,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:01,337 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:02,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:01:02,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:01:02,598 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:01:03,228 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:01:03,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:03,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:05,231 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:01:05,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:05,518 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:06,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:07,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:07,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:10,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:10,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:11,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:12,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:12,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:15,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:15,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:16,593 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:17,236 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:01:17,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:01:17,589 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:01:17,598 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:01:17,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:17,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:20,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:20,955 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:21,812 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:22,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:22,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:25,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:25,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:26,813 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:27,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:27,821 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:31,240 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:01:31,371 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:01:31,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:31,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:32,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:01:32,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:32,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:01:32,598 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:01:32,831 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:01:34,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:34,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:36,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:36,219 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:37,832 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:39,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:39,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:41,243 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:01:41,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:41,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:42,938 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:43,370 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:43,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:46,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:46,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:47,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:01:47,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:01:47,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:01:48,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:48,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:48,406 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:49,245 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:01:50,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:50,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:53,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:53,412 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:01:53,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:55,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:55,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:57,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:57,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:01:58,413 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:00,465 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:00,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:02,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:02,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:02,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:02:02,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:02:02,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:02:02,833 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:02:03,249 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:02:03,834 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:04,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:04,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:08,120 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:02:08,148 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:08,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:08,834 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:11,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:11,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:13,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:13,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:13,835 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:15,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:15,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:17,254 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:02:17,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:02:17,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:02:17,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:02:18,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:18,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:19,749 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:20,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:20,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:22,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:22,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:24,750 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:25,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:25,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:27,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:27,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:29,257 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:02:29,332 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:29,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:30,220 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:32,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:32,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:32,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:02:32,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:02:32,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:02:32,839 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:02:34,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:34,458 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:35,840 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:36,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:36,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:39,625 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:39,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:40,841 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:41,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:41,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:43,261 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:02:45,503 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:02:45,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:45,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:46,490 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:47,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:02:47,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:02:47,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:02:48,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:48,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:50,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:50,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:51,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:53,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:53,864 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:55,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:55,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:57,047 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:02:57,265 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:02:57,923 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:02:57,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:01,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:01,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:02,048 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:02,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:03:02,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:03:02,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:03:02,836 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:03:03,076 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:03,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:05,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:05,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:07,838 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:08,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:08,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:09,268 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:03:10,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:10,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:12,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:12,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:13,634 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:15,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:15,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:17,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:17,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:17,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:03:17,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:03:17,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:03:18,779 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:19,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:19,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:22,833 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:03:22,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:23,272 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:03:23,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:24,032 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:25,789 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:25,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:27,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:27,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:29,034 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:29,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:29,864 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:32,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:03:32,590 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:03:32,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:03:32,838 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:03:32,854 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:32,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:34,839 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:34,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:34,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:36,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:36,926 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:37,275 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:03:39,953 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:39,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:40,431 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:41,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:41,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:44,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:44,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:45,432 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:47,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:47,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:47,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:03:47,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:03:47,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:03:49,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:49,081 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:50,919 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:51,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:51,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:53,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:53,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:53,279 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:03:55,920 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:03:56,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:56,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:59,822 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:03:59,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:03:59,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:00,921 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:01,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:01,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:02,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:04:02,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:04:02,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:04:02,839 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:04:03,282 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:04:04,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:04,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:06,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:06,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:06,841 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:08,845 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:08,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:11,842 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:11,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:11,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:13,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:13,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:16,017 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:16,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:17,170 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:17,287 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:04:17,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:04:17,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:04:17,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:04:19,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:19,159 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:21,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:21,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:22,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:24,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:24,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:25,289 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:04:26,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:26,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:27,853 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:28,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:28,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:31,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:31,260 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:31,291 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:04:32,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:04:32,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:04:32,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:04:32,841 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:04:33,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:33,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:33,842 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:37,029 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:04:37,052 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:37,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:38,842 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:40,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:40,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:41,294 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:04:42,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:42,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:44,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:44,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:44,626 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:47,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:47,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:47,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:04:47,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:04:47,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:04:49,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:49,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:49,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:51,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:51,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:53,297 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:04:54,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:54,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:54,843 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:04:56,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:56,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:58,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:58,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:04:59,844 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:01,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:01,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:02,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:05:02,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:05:02,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:05:02,842 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:05:03,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:03,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:05,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:05,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:05,844 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:07,301 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:05:08,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:08,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:10,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:10,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:11,066 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:14,245 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:05:14,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:14,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:16,067 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:17,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:17,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:17,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:05:17,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:05:17,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:05:19,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:19,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:21,201 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:21,305 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:05:21,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:21,465 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:24,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:24,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:26,202 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:26,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:26,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:28,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:28,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:31,203 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:31,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:31,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:32,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:05:32,591 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:05:32,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:05:32,845 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:05:33,309 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:05:33,539 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:33,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:35,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:35,580 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:36,846 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:38,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:38,688 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:40,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:40,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:41,847 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:43,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:43,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:45,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:45,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:47,301 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:47,313 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:05:47,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:05:47,592 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:05:47,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:05:47,745 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:47,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:51,448 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:05:51,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:51,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:52,789 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:54,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:54,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:56,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:56,591 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:57,790 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:05:58,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:58,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:05:59,316 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:06:01,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:01,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:02,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:06:02,592 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:06:02,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:06:02,846 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:06:02,848 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:03,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:03,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:05,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:05,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:07,848 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:08,876 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:08,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:10,918 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:10,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:12,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:12,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:13,319 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:06:13,513 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:16,076 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:16,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:17,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:06:17,592 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:06:17,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:06:18,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:18,118 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:18,790 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:20,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:20,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:23,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:23,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:23,791 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:25,322 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:06:25,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:25,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:29,085 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:06:29,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:29,120 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:29,638 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:31,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:31,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:32,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:06:32,592 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:06:32,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:06:32,848 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:06:33,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:33,854 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:34,849 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:35,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:35,894 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:37,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:37,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:39,326 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:06:40,046 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:40,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:40,948 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:42,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:43,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:45,046 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:45,905 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:45,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:47,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:06:47,592 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:06:47,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:06:47,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:47,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:50,824 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:50,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:50,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:52,902 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:52,918 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:53,330 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:06:55,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:55,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:56,751 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:06:57,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:06:58,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:00,035 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:00,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:01,752 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:02,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:07:02,592 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:07:02,605 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:07:02,849 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:07:03,138 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:03,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:06,789 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:07:06,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:06,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:07,445 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:08,335 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:07:09,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:09,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:11,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:11,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:12,446 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:14,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:14,937 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:16,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:17,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:17,447 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:17,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:07:17,598 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:07:17,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:07:19,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:20,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:20,339 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:07:22,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:22,079 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:23,041 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:25,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:25,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:27,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:27,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:28,042 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:29,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:29,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:32,043 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:32,059 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:32,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:07:32,592 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:07:32,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:07:32,851 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:07:33,852 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:34,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:34,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:34,344 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:07:37,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:37,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:38,853 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:39,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:39,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:42,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:42,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:43,854 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:45,741 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:07:45,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:45,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:47,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:07:47,597 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:07:47,606 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:07:47,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:47,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:48,350 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:07:49,744 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:50,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:50,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:52,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:52,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:54,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:07:55,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:55,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:57,465 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:07:57,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:00,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:00,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:00,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:02,354 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:08:02,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:02,592 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:08:02,593 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:08:02,602 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:08:02,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:02,854 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:08:04,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:04,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:05,856 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:07,642 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:08:07,644 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 16:08:07,644 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 16:08:07,645 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 16:08:07,645 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 16:08:07,645 DEBUG SenderThread:1483352 [sender.py:send():382] send: metric -2024-03-27 16:08:07,645 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:08:07,645 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:08:07,646 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:08:07,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:07,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:08,356 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:08:08,356 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:08:09,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:09,779 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:10,356 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:08:11,648 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:11,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:11,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:14,821 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:14,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:16,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:16,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:17,554 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:17,769 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:08:17,800 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:08:17,800 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:08:18,360 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:08:18,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:18,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:22,390 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:08:23,312 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:23,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:24,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:27,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:27,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:28,313 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:29,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:29,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:32,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:32,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:32,685 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:08:32,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:08:32,686 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:08:33,147 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:08:34,153 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:34,365 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/config.yaml -2024-03-27 16:08:34,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:34,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:37,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:37,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:39,331 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:39,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:41,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:43,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:43,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:44,332 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:46,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:46,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:47,685 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:08:47,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:08:47,686 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:08:48,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:48,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:48,369 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:08:49,913 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:51,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:51,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:53,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:53,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:54,370 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:08:55,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:55,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:55,583 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:08:58,372 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:08:58,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:08:58,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:00,658 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:00,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:00,946 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:02,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:09:02,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:09:02,687 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:09:02,905 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:09:04,374 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:04,428 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:09:04,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:04,458 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:06,906 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:07,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:07,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:09,606 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:09,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:10,375 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:11,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:11,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:12,391 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:14,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:14,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:16,377 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:16,864 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:16,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:17,515 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:17,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:09:17,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:09:17,687 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:09:18,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:18,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:22,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:22,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:22,379 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:22,878 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:24,167 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:24,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:26,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:26,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:28,381 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:28,750 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:29,291 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:29,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:31,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:31,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:32,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:09:32,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:09:32,686 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:09:32,858 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:09:33,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:33,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:33,859 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:34,383 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:36,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:36,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:38,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:38,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:38,990 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:40,384 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:42,911 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:09:43,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:43,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:43,991 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:45,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:45,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:46,386 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:47,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:09:47,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:09:47,687 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:09:47,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:47,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:49,845 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:50,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:50,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:52,388 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:09:52,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:52,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:54,971 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:54,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:55,263 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:09:58,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:58,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:09:58,390 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:00,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:00,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:00,388 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:02,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:02,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:02,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:10:02,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:10:02,687 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:10:02,908 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:10:04,392 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:05,539 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:05,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:05,910 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:07,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:07,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:09,706 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:09,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:10,394 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:11,631 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:12,837 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:12,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:14,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:14,902 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:16,396 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:16,857 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:16,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:16,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:17,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:10:17,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:10:17,687 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:10:20,627 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:10:20,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:20,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:22,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:22,398 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:23,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:23,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:25,717 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:25,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:27,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:27,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:27,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:30,403 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:30,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:30,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:32,417 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:32,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:10:32,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:10:32,687 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:10:32,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:32,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:32,893 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:10:34,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:34,903 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:36,404 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:37,645 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:37,948 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:37,972 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:39,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:40,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:42,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:42,079 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:42,406 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:42,925 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:45,138 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:45,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:46,408 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:47,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:47,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:47,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:10:47,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:10:47,687 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:10:48,409 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:48,874 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:49,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:49,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:51,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:51,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:52,410 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:53,981 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:10:54,411 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:10:54,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:54,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:58,317 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:10:58,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:58,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:10:59,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:00,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:00,412 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:00,421 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:02,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:11:02,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:11:02,687 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:11:02,864 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:11:03,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:03,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:04,866 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:05,010 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:11:05,012 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:11:05,012 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:11:05,013 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:11:05,414 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:11:05,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:05,557 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:06,414 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:07,584 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:07,602 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:10,013 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:10,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:10,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:12,416 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:12,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:12,921 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:14,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:14,961 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:15,228 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:16,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:17,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:11:17,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:11:17,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:11:18,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:20,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:20,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:20,418 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:20,911 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:22,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:22,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:25,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:25,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:26,420 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:26,614 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:27,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:27,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:29,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:29,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:31,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:31,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:31,974 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:32,422 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:32,686 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:11:32,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:11:32,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:11:32,917 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:11:35,037 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:11:35,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:36,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:37,405 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:38,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:38,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:38,424 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:40,428 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:41,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:42,406 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:43,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:43,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:44,426 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:45,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:45,903 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:47,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:11:47,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:11:47,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:11:47,877 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:49,021 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:49,052 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:50,428 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:51,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:51,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:53,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:53,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:53,307 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:11:56,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:56,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:56,430 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:11:58,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:58,225 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:11:58,799 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:00,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:00,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:02,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:02,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:02,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:12:02,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:12:02,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:12:02,867 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:12:03,868 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:04,432 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:05,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:05,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:07,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:07,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:09,307 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:09,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:09,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:10,434 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:13,082 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:12:13,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:13,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:14,308 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:16,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:16,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:16,436 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:17,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:12:17,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:12:17,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:12:18,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:18,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:19,894 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:20,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:21,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:22,438 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:23,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:23,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:25,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:26,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:26,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:28,440 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:28,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:28,810 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:30,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:30,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:30,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:32,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:12:32,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:12:32,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:12:32,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:32,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:32,900 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:12:34,442 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:35,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:35,893 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:35,901 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:37,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:37,926 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:39,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:39,955 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:40,444 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:40,946 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:43,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:43,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:45,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:45,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:46,079 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:46,446 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:47,224 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:47,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:47,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:12:47,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:12:47,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:12:50,909 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:12:50,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:50,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:51,218 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:52,448 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:12:53,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:53,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:55,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:55,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:56,219 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:12:58,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:58,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:12:58,450 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:01,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:01,158 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:01,344 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:02,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:13:02,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:13:02,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:13:02,872 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:13:03,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:03,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:04,452 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:05,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:05,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:06,489 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:08,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:08,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:10,204 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:10,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:10,453 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:11,618 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:12,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:12,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:15,165 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:15,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:16,455 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:16,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:17,391 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:17,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:17,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:13:17,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:13:17,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:13:20,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:20,467 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:21,873 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:22,457 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:22,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:22,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:24,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:24,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:26,874 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:28,459 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:28,673 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:13:29,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:29,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:31,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:31,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:32,001 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:32,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:13:32,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:13:32,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:13:32,873 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:13:34,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:34,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:36,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:36,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:37,138 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:37,463 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:39,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:39,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:40,372 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:13:40,373 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:13:40,373 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:13:40,374 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:13:40,464 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:13:41,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:41,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:42,374 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:43,467 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:44,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:44,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:46,757 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:46,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:47,575 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:47,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:13:47,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:13:47,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:13:49,469 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:49,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:49,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:51,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:51,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:52,917 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:53,905 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:53,934 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:55,471 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:13:56,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:56,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:57,986 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:13:58,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:13:58,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:00,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:00,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:01,473 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:02,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:14:02,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:14:02,688 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:14:02,907 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:14:03,801 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:03,838 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:03,908 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:07,475 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:07,563 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:14:07,585 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:07,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:09,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:10,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:10,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:12,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:12,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:14,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:14,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:15,348 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:15,478 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:17,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:14:17,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:14:17,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:14:17,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:17,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:19,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:19,837 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:20,819 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:21,480 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:21,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:21,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:24,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:24,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:26,161 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:26,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:26,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:27,482 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:30,008 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:30,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:31,162 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:32,052 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:32,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:32,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:14:32,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:14:32,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:14:32,877 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:14:33,484 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:35,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:35,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:36,879 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:37,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:37,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:39,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:39,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:39,486 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:42,135 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:42,485 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:42,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:46,252 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:14:46,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:46,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:47,489 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:47,510 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:47,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:14:47,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:14:47,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:14:48,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:48,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:50,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:50,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:52,966 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:53,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:53,491 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:53,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:55,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:55,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:57,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:57,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:58,392 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:14:59,493 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:14:59,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:14:59,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:02,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:15:02,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:15:02,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:15:02,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:02,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:02,878 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:15:03,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:04,745 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:04,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:05,495 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:06,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:06,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:08,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:08,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:08,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:11,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:11,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:13,497 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:13,893 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:13,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:14,331 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:15,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:15,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:17,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:15:17,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:15:17,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:15:19,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:19,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:19,499 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:21,159 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:22,780 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:15:22,810 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:22,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:24,845 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:24,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:25,501 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:26,585 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:27,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:27,933 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:29,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:29,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:31,503 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:32,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:32,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:32,122 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:32,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:15:32,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:15:32,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:15:32,896 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:15:35,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:35,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:37,221 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:37,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:37,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:39,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:39,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:39,506 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:42,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:42,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:42,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:44,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:44,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:45,508 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:46,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:46,604 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:47,688 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:15:47,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:15:47,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:15:47,830 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:49,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:49,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:51,510 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:51,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:51,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:53,392 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:53,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:53,804 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:55,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:55,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:57,512 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:15:58,494 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:15:59,598 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:15:59,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:15:59,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:02,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:16:02,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:16:02,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:16:02,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:02,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:02,896 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:16:03,513 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:03,897 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:04,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:04,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:06,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:06,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:09,515 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:09,778 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:09,993 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:10,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:12,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:12,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:14,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:14,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:14,907 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:15,517 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:17,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:17,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:17,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:16:17,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:16:17,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:16:19,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:19,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:19,915 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:20,124 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:16:20,125 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:16:20,126 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:16:20,126 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:16:20,519 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:16:21,219 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:21,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:21,519 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:24,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:24,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:25,128 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:26,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:26,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:27,521 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:28,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:28,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:30,182 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:31,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:31,585 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:32,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:16:32,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:16:32,689 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:16:32,883 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:16:33,606 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:33,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:35,524 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:35,884 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:37,358 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:16:37,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:37,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:40,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:40,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:41,446 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:41,526 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:42,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:42,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:44,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:44,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:46,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:46,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:46,648 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:47,528 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:47,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:16:47,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:16:47,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:16:49,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:49,836 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:51,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:51,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:51,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:53,530 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:16:53,888 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:53,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:56,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:16:57,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:57,074 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:59,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:59,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:16:59,532 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:01,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:01,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:01,866 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:02,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:17:02,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:17:02,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:17:02,884 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:17:03,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:03,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:05,534 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:06,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:06,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:06,989 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:08,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:08,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:11,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:11,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:11,536 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:12,139 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:15,101 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:17:15,123 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:15,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:17,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:17,167 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:17,538 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:17,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:17:17,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:17:17,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:17:17,880 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:19,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:19,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:19,539 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:22,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:22,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:23,034 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:23,540 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:24,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:24,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:25,541 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:26,411 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:26,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:28,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:29,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:29,511 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:29,543 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:31,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:31,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:32,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:17:32,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:17:32,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:17:32,886 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:17:33,544 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:33,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:33,581 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:33,887 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:36,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:36,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:38,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:38,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:39,362 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:39,546 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:40,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:40,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:42,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:42,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:44,491 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:45,549 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:45,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:45,821 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:47,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:17:47,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:17:47,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:17:47,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:47,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:49,627 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:51,551 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:51,637 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:17:51,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:51,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:54,628 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:17:54,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:54,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:56,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:56,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:57,553 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:17:58,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:58,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:17:59,753 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:01,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:01,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:02,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:18:02,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:18:02,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:18:02,888 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:18:03,555 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:03,827 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:03,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:04,889 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:07,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:07,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:09,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:09,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:09,557 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:10,019 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:11,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:11,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:14,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:14,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:15,143 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:15,559 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:16,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:16,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:17,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:18:17,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:18:17,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:18:18,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:18,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:20,274 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:21,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:21,424 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:21,563 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:23,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:23,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:25,275 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:25,518 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:25,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:29,431 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:18:29,565 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:30,418 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:30,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:30,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:32,489 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:32,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:32,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:18:32,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:18:32,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:18:32,890 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:18:34,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:34,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:35,567 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:35,892 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:37,561 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:37,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:38,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:18:38,759 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:18:38,759 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:18:38,760 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:18:39,569 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:18:39,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:39,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:41,569 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:41,627 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:41,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:41,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:44,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:44,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:46,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:46,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:46,836 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:47,571 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:47,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:18:47,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:18:47,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:18:48,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:48,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:51,957 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:52,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:52,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:53,573 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:18:54,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:54,081 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:56,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:56,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:56,957 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:18:59,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:59,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:18:59,575 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:19:01,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:01,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:02,094 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:02,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:19:02,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:19:02,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:19:02,891 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:19:03,183 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:03,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:05,577 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:19:06,802 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:19:06,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:06,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:07,216 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:09,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:09,854 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:11,579 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:19:11,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:11,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:12,497 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:13,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:13,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:15,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:15,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:17,581 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:19:17,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:17,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:19:17,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:19:17,690 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:19:18,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:18,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:21,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:21,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:22,769 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:23,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:23,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:23,584 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:19:26,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:26,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:27,769 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:28,222 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:28,232 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:29,585 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:19:30,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:30,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:32,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:19:32,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:19:32,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:19:32,926 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:19:32,927 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:33,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:33,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:35,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:35,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:35,587 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:19:37,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:37,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:38,016 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:40,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:40,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:41,589 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:19:43,353 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:44,627 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:19:44,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:44,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:47,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:19:47,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:19:47,692 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:19:47,773 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:47,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:48,817 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:49,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:49,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:52,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:52,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:53,823 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:54,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:54,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:58,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:58,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:19:59,487 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:19:59,594 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:00,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:00,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:01,595 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:02,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:20:02,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:20:02,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:20:02,895 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:20:03,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:03,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:04,888 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:05,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:05,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:07,597 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:08,445 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:08,467 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:09,889 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:10,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:10,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:12,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:12,569 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:13,599 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:14,982 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:15,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:15,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:17,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:17,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:17,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:20:17,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:20:17,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:20:19,601 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:19,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:19,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:20,853 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:24,181 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:20:24,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:24,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:25,603 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:26,229 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:26,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:26,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:28,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:28,674 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:31,363 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:31,605 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:31,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:31,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:32,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:20:32,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:20:32,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:20:32,906 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:20:33,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:33,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:35,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:35,921 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:36,498 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:37,607 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:38,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:38,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:41,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:41,041 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:42,210 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:43,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:43,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:43,608 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:45,609 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:46,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:46,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:47,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:20:47,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:20:47,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:20:47,936 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:48,222 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:48,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:49,610 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:50,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:50,293 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:51,611 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:53,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:53,362 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:53,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:53,612 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:20:55,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:55,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:58,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:58,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:20:59,186 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:20:59,614 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:02,281 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:21:02,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:02,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:02,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:21:02,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:21:02,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:21:02,898 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:21:04,182 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:21:04,184 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:21:04,184 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:21:04,185 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:21:04,616 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:21:05,185 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:05,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:05,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:05,617 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:07,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:07,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:10,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:10,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:11,072 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:11,620 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:12,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:12,585 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:15,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:15,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:16,073 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:17,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:17,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:17,621 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:17,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:21:17,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:21:17,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:21:20,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:20,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:21,851 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:22,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:22,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:23,623 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:25,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:25,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:26,851 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:27,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:27,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:29,625 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:30,953 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:30,972 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:31,960 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:32,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:21:32,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:21:32,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:21:32,900 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:21:33,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:33,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:35,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:35,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:35,627 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:37,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:38,035 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:38,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:41,631 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:41,792 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:21:41,817 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:41,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:43,267 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:43,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:43,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:46,837 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:46,847 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:47,632 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:47,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:21:47,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:21:47,691 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:21:48,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:48,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:50,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:50,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:51,373 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:53,634 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:21:54,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:54,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:56,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:56,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:56,373 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:21:58,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:58,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:21:59,636 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:01,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:01,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:02,222 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:02,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:22:02,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:22:02,692 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:22:02,901 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:22:03,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:03,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:05,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:05,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:05,638 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:07,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:07,903 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:08,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:09,639 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:10,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:10,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:12,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:12,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:13,671 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:15,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:15,322 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:15,641 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:17,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:22:17,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:22:17,692 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:22:18,920 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:19,200 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:22:19,224 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:19,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:21,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:21,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:21,643 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:24,197 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:24,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:24,233 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:26,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:26,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:27,645 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:28,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:28,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:30,079 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:31,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:31,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:32,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:22:32,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:22:32,692 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:22:32,902 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:22:33,417 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:33,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:33,647 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:35,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:35,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:35,904 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:38,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:38,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:39,649 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:40,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:40,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:41,022 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:42,604 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:42,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:45,651 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:45,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:45,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:46,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:47,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:22:47,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:22:47,692 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:22:47,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:47,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:49,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:49,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:51,653 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:52,543 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:52,886 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:52,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:56,643 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:22:56,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:56,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:57,655 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:22:57,951 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:22:58,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:22:58,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:01,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:01,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:02,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:23:02,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:23:02,692 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:23:02,971 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:23:02,972 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:03,657 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:03,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:03,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:05,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:05,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:07,659 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:08,640 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:08,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:08,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:10,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:11,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:13,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:13,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:13,796 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:15,661 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:15,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:16,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:17,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:23:17,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:23:17,692 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:23:18,059 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:18,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:18,852 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:20,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:20,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:21,662 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:23,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:23,293 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:24,002 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:25,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:25,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:27,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:27,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:27,664 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:29,067 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:30,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:30,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:31,819 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:23:31,820 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:23:31,821 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:23:31,821 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:23:32,666 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:23:32,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:23:32,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:23:32,693 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:23:33,069 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:23:33,666 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:34,073 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:34,427 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:23:34,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:34,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:37,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:37,647 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:39,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:39,668 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:39,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:39,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:41,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:41,735 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:44,710 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:44,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:44,764 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:45,670 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:46,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:46,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:47,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:23:47,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:23:47,693 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:23:48,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:48,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:50,713 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:51,672 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:51,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:51,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:53,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:53,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:55,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:23:55,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:55,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:57,674 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:23:58,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:23:58,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:00,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:00,969 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:00,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:02,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:24:02,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:24:02,693 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:24:02,909 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:24:03,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:03,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:03,676 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:05,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:05,838 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:06,912 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:07,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:07,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:09,678 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:11,664 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:24:11,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:11,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:11,964 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:14,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:14,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:15,681 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:16,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:16,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:17,250 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:17,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:24:17,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:24:17,693 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:24:18,871 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:18,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:21,682 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:21,923 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:21,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:23,046 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:23,960 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:23,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:25,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:26,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:27,684 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:28,265 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:29,059 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:29,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:31,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:31,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:32,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:24:32,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:24:32,693 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:24:32,911 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:24:33,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:33,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:33,686 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:33,912 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:36,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:36,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:38,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:38,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:39,688 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:39,848 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:40,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:40,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:43,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:43,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:45,470 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:45,505 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:45,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:45,690 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:47,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:24:47,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:24:47,693 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:24:49,190 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:24:49,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:49,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:51,399 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:51,691 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:52,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:52,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:54,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:54,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:56,458 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:56,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:57,220 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:24:57,692 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:24:59,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:24:59,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:01,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:01,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:02,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:25:02,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:25:02,694 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:25:02,870 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:02,912 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:25:03,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:03,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:03,694 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:06,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:06,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:07,924 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:08,364 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:08,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:09,695 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:10,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:10,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:13,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:13,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:13,463 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:13,697 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:15,489 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:15,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:17,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:17,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:17,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:25:17,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:25:17,694 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:25:19,412 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:19,698 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:20,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:20,505 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:22,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:22,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:24,962 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:25,700 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:26,255 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:25:26,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:26,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:29,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:29,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:30,905 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:31,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:31,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:31,702 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:32,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:25:32,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:25:32,694 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:25:32,926 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:25:33,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:33,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:36,584 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:36,621 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:36,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:37,704 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:38,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:38,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:41,921 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:41,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:42,604 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:43,705 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:43,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:43,961 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:45,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:45,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:47,693 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:25:47,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:25:47,694 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:25:48,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:48,968 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:49,707 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:50,613 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:50,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:51,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:53,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:53,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:55,708 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:25:55,767 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:55,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:56,125 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:25:57,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:57,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:59,134 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:25:59,136 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:25:59,136 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:25:59,136 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:25:59,709 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:25:59,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:25:59,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:01,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:01,710 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:02,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:26:02,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:26:02,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:26:02,916 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:26:03,580 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:26:03,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:03,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:05,711 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:06,664 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:06,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:06,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:08,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:08,735 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:10,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:10,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:11,713 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:12,605 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:13,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:13,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:15,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:15,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:17,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:26:17,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:26:17,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:26:17,714 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:17,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:17,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:17,891 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:20,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:20,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:22,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:23,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:23,078 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:23,716 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:25,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:25,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:28,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:28,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:28,929 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:29,718 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:30,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:30,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:32,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:32,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:32,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:26:32,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:26:32,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:26:32,917 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:26:34,608 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:35,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:35,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:35,720 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:37,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:37,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:40,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:41,163 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:26:41,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:41,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:41,722 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:44,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:44,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:45,724 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:46,037 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:46,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:46,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:47,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:26:47,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:26:47,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:26:48,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:48,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:51,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:51,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:51,359 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:53,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:53,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:53,726 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:56,360 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:26:56,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:56,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:57,727 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:26:58,611 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:26:58,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:01,536 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:01,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:01,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:02,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:27:02,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:27:02,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:27:02,919 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:27:03,729 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:03,768 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:03,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:06,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:06,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:06,921 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:08,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:08,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:09,731 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:12,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:12,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:12,287 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:14,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:14,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:15,733 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:17,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:17,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:17,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:27:17,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:27:17,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:27:17,875 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:21,159 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:27:21,179 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:21,189 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:21,735 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:23,081 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:24,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:24,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:26,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:26,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:27,737 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:28,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:28,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:28,664 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:31,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:31,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:32,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:27:32,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:27:32,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:27:32,921 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:27:33,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:33,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:33,739 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:33,922 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:36,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:36,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:38,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:38,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:39,153 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:39,741 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:40,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:40,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:43,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:43,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:43,742 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:45,092 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:45,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:45,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:47,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:27:47,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:27:47,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:27:48,755 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:48,779 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:49,745 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:50,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:50,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:50,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:52,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:52,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:55,747 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:27:55,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:55,789 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:56,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:27:59,514 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:27:59,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:27:59,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:01,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:01,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:01,750 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:02,531 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:02,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:28:02,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:28:02,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:28:02,922 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:28:04,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:04,569 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:06,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:06,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:07,752 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:08,099 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:09,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:09,684 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:11,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:11,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:13,754 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:13,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:13,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:14,071 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:16,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:16,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:17,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:28:17,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:28:17,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:28:18,684 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:18,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:19,756 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:21,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:21,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:21,933 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:23,645 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:28:23,646 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:28:23,647 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:28:23,647 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:28:23,758 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:28:23,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:23,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:25,759 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:25,877 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:25,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:27,649 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:28,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:28,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:31,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:31,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:31,760 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:32,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:28:32,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:28:32,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:28:32,859 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:32,924 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:28:33,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:33,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:35,762 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:36,811 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:28:36,838 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:36,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:38,144 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:39,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:39,810 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:41,764 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:41,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:41,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:43,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:43,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:44,009 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:47,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:47,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:47,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:28:47,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:28:47,695 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:28:47,766 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:49,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:49,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:50,835 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:51,100 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:51,112 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:53,768 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:28:54,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:54,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:55,928 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:28:56,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:56,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:58,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:58,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:28:59,769 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:01,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:01,325 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:01,554 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:02,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:29:02,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:29:02,696 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:29:03,293 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:29:03,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:03,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:05,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:05,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:06,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:06,772 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:08,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:08,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:10,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:10,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:12,099 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:12,774 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:14,435 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:29:14,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:14,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:17,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:17,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:17,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:29:17,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:29:17,696 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:29:17,842 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:18,775 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:19,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:19,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:21,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:21,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:22,777 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:23,604 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:24,561 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:24,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:26,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:26,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:28,779 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:29,547 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:29,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:29,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:31,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:31,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:32,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:29:32,960 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:29:32,960 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:29:33,149 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:29:33,920 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:33,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:34,783 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:35,198 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:35,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:35,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:39,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:39,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:40,786 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:40,892 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:41,090 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:41,109 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:43,148 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:43,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:46,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:46,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:46,788 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:46,830 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:47,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:29:47,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:29:47,696 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:29:48,165 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:48,183 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:51,884 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:29:51,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:51,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:52,423 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:52,790 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:55,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:55,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:57,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:57,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:57,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:29:58,793 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:29:59,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:29:59,120 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:02,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:02,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:02,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:30:02,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:30:02,696 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:30:02,794 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:02,888 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:02,929 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:30:04,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:04,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:06,561 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:06,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:07,930 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:08,796 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:09,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:09,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:11,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:11,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:12,939 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:14,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:14,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:14,798 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:16,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:16,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:17,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:30:17,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:30:17,696 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:30:18,882 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:18,894 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:18,931 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:20,800 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:21,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:21,877 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:23,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:23,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:24,435 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:25,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:25,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:26,802 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:29,693 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:30:29,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:29,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:30,376 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:32,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:32,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:32,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:30:32,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:30:32,696 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:30:32,804 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:32,931 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:30:34,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:34,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:35,933 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:36,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:36,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:38,806 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:39,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:39,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:41,876 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:41,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:41,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:43,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:43,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:44,808 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:45,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:45,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:47,466 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:47,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:30:47,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:30:47,697 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:30:48,479 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:30:48,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:48,933 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:49,867 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:30:49,867 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:30:49,868 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:30:50,810 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:30:50,810 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:50,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:50,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:51,811 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:52,869 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:52,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:53,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:54,813 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:30:56,148 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:56,158 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:57,994 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:30:58,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:30:58,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:00,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:00,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:00,815 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:02,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:02,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:02,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:31:02,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:31:02,697 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:31:02,933 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:31:03,934 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:06,403 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:31:06,817 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:06,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:06,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:08,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:08,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:09,681 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:12,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:12,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:12,819 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:14,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:14,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:15,272 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:16,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:16,137 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:17,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:31:17,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:31:17,697 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:31:18,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:18,173 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:18,821 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:20,853 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:21,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:21,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:23,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:23,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:24,822 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:25,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:25,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:26,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:27,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:28,627 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:30,648 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:30,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:30,824 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:32,654 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:32,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:32,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:32,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:31:32,696 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:31:32,697 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:31:32,936 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:31:34,825 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:35,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:35,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:37,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:37,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:38,227 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:39,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:39,786 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:40,827 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:43,587 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:31:43,615 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:43,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:44,173 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:46,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:46,604 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:46,829 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:47,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:31:47,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:31:47,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:31:48,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:48,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:49,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:50,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:50,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:52,831 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:31:53,750 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:53,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:55,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:55,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:55,853 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:31:57,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:57,835 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:31:58,833 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:00,971 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:00,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:01,807 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:02,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:32:02,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:32:02,697 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:32:02,938 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:32:03,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:03,017 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:04,834 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:05,041 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:05,052 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:07,318 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:07,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:07,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:09,920 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:09,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:10,836 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:11,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:11,965 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:13,261 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:14,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:15,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:16,838 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:17,041 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:17,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:17,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:32:17,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:32:17,697 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:32:20,401 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:20,892 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:32:20,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:20,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:22,841 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:23,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:23,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:25,821 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:25,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:25,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:26,842 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:27,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:28,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:30,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:30,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:31,347 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:32,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:32:32,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:32:32,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:32:32,844 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:32,939 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:32:33,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:33,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:35,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:35,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:36,941 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:38,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:38,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:38,846 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:40,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:40,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:42,282 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:42,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:42,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:44,849 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:45,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:45,541 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:47,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:47,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:47,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:32:47,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:32:47,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:32:47,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:49,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:49,606 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:50,851 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:52,669 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:52,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:53,659 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:32:54,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:54,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:56,853 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:32:58,451 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:32:58,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:58,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:32:59,210 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:01,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:01,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:02,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:33:02,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:33:02,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:33:02,855 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:02,941 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:33:03,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:03,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:04,942 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:05,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:05,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:08,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:08,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:08,856 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:10,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:10,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:10,814 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:12,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:12,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:12,858 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:33:12,860 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:33:12,860 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:33:12,860 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:33:13,858 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:33:14,858 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:15,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:15,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:15,861 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:17,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:17,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:17,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:33:17,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:33:17,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:33:19,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:19,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:20,860 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:21,660 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:22,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:22,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:24,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:24,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:24,862 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:26,860 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:26,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:27,498 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:29,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:29,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:30,865 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:31,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:31,926 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:32,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:33:32,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:33:32,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:33:32,825 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:32,943 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:33:35,732 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:33:35,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:35,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:36,866 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:38,043 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:38,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:38,745 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:40,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:40,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:42,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:42,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:42,868 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:43,989 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:45,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:45,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:47,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:33:47,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:33:47,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:33:47,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:47,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:48,870 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:49,847 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:49,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:49,884 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:53,097 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:53,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:54,872 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:33:55,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:55,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:55,502 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:33:57,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:57,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:59,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:33:59,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:00,874 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:01,010 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:01,935 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:01,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:02,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:34:02,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:34:02,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:34:02,949 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:34:03,968 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:03,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:06,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:06,825 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:06,876 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:07,003 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:08,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:08,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:12,552 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:34:12,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:12,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:12,944 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:13,879 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:15,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:15,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:17,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:17,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:17,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:34:17,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:34:17,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:34:17,880 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:19,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:19,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:20,658 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:22,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:22,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:23,882 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:24,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:24,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:26,472 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:27,792 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:27,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:29,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:29,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:29,886 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:31,864 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:31,876 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:32,034 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:32,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:34:32,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:34:32,698 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:34:32,946 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:34:34,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:34,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:35,889 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:36,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:36,790 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:37,979 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:38,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:38,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:41,891 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:41,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:41,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:43,654 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:44,011 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:44,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:47,090 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:47,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:47,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:34:47,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:34:47,699 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:34:47,897 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:48,906 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:50,682 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:34:50,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:50,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:52,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:52,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:53,899 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:54,269 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:34:55,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:55,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:57,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:57,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:34:59,901 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:34:59,921 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:00,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:00,902 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:02,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:35:02,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:35:02,699 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:35:02,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:02,948 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:35:02,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:04,949 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:04,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:05,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:05,903 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:07,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:07,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:09,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:09,868 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:09,904 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:10,399 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:11,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:12,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:15,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:15,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:15,906 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:16,344 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:17,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:17,079 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:17,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:35:17,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:35:17,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:35:20,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:20,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:21,909 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:21,920 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:22,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:22,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:25,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:25,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:27,697 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:27,912 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:28,790 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:35:28,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:28,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:31,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:31,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:32,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:35:32,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:35:32,699 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:35:32,864 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:32,949 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:35:33,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:33,770 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:33,914 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:35,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:35,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:37,147 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:35:37,149 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:35:37,149 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:35:37,151 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:35:37,915 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:35:38,152 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:38,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:38,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:38,915 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:39,916 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:40,845 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:40,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:42,902 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:42,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:44,015 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:45,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:45,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:45,918 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:47,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:35:47,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:35:47,699 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:35:47,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:47,960 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:49,580 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:49,920 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:50,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:50,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:53,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:53,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:55,112 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:55,128 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:55,529 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:35:55,922 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:35:57,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:57,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:59,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:35:59,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:00,924 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:01,924 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:02,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:02,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:02,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:36:02,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:36:02,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:36:02,951 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:36:06,045 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:36:06,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:06,078 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:06,849 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:07,928 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:08,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:08,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:11,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:11,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:12,407 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:13,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:13,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:13,930 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:15,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:15,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:17,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:36:17,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:36:17,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:36:17,869 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:18,477 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:18,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:19,932 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:20,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:20,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:22,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:22,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:22,870 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:25,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:25,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:25,934 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:27,877 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:27,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:28,261 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:29,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:29,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:31,936 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:32,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:36:32,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:36:32,699 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:36:32,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:32,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:32,953 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:36:33,954 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:34,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:34,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:36,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:36,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:37,938 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:39,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:39,918 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:39,927 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:41,939 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:43,672 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:36:43,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:43,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:45,656 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:45,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:45,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:47,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:36:47,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:36:47,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:36:47,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:47,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:47,941 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:50,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:50,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:51,795 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:52,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:52,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:53,943 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:36:54,773 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:54,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:57,481 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:36:57,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:57,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:59,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:59,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:36:59,944 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:01,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:01,902 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:02,630 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:02,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:37:02,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:37:02,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:37:02,954 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:37:04,986 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:04,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:05,946 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:07,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:07,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:08,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:09,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:09,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:11,948 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:12,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:12,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:14,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:14,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:14,322 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:16,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:16,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:17,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:37:17,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:37:17,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:37:17,950 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:19,979 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:37:20,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:20,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:20,936 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:22,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:22,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:23,952 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:25,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:25,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:26,060 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:27,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:27,059 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:29,954 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:30,063 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:30,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:32,048 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:32,208 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:32,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:32,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:37:32,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:37:32,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:37:32,956 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:37:35,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:35,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:35,957 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:37,633 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:37,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:37,750 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:39,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:39,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:41,958 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:42,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:42,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:42,848 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:44,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:44,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:46,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:46,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:47,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:37:47,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:37:47,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:37:47,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:47,960 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:49,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:50,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:52,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:52,043 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:53,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:53,962 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:37:54,071 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:54,095 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:58,638 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:37:59,017 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:59,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:37:59,639 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:37:59,965 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:01,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:01,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:02,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:38:02,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:38:02,700 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:38:02,957 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:38:03,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:03,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:03,118 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:38:03,119 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:38:03,119 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:38:03,120 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:38:03,966 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:38:04,966 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:05,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:05,121 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:05,130 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:05,967 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:08,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:08,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:10,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:10,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:11,001 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:11,969 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:12,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:12,421 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:14,445 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:15,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:15,971 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:16,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:17,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:17,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:17,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:38:17,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:38:17,701 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:38:20,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:20,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:21,973 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:22,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:22,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:22,657 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:25,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:25,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:27,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:27,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:27,975 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:28,240 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:30,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:30,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:32,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:32,602 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:32,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:38:32,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:38:32,701 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:38:32,959 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:38:33,960 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:33,976 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:35,915 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:38:35,936 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:35,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:38,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:38,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:39,846 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:39,978 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:40,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:40,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:42,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:42,735 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:45,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:45,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:45,787 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:45,981 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:47,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:47,541 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:47,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:38:47,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:38:47,702 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:38:50,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:50,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:51,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:51,982 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:38:52,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:52,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:55,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:55,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:57,259 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:38:57,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:57,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:38:57,985 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:00,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:00,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:02,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:02,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:02,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:39:02,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:39:02,701 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:39:02,905 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:02,960 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:39:03,986 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:04,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:05,008 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:07,041 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:07,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:07,988 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:08,423 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:10,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:10,065 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:13,645 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:39:13,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:13,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:13,990 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:14,339 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:16,768 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:16,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:17,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:39:17,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:39:17,702 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:39:18,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:18,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:19,929 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:19,992 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:21,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:21,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:23,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:23,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:25,762 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:25,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:25,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:25,994 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:28,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:28,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:30,561 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:30,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:31,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:31,995 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:32,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:39:32,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:39:32,701 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:39:32,962 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:39:33,653 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:33,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:35,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:35,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:37,263 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:37,997 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:38,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:40,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:42,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:42,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:43,184 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:43,999 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:45,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:45,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:47,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:39:47,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:39:47,702 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:39:48,000 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:48,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:48,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:48,847 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:50,663 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:50,676 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:54,002 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:39:54,069 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:39:54,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:54,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:54,640 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:39:56,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:56,836 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:58,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:39:58,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:00,004 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:00,245 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:01,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:01,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:02,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:40:02,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:40:02,702 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:40:02,963 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:40:03,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:03,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:06,006 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:06,165 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:06,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:06,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:08,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:08,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:11,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:11,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:11,875 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:12,008 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:13,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:13,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:16,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:16,414 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:17,622 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:17,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:40:17,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:40:17,701 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:40:18,010 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:18,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:18,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:21,225 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:21,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:22,640 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:23,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:24,012 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:24,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:26,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:26,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:27,424 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:40:27,424 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:40:27,425 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:40:27,425 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:40:28,013 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:40:28,014 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:28,426 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:29,014 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:29,322 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:29,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:32,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:40:32,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:40:32,702 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:40:32,936 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:40:32,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:32,965 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:40:32,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:34,016 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:34,413 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:35,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:35,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:37,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:37,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:40,018 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:40,106 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:40,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:40,827 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:42,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:42,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:45,106 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:45,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:45,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:46,021 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:47,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:40:47,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:40:47,702 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:40:47,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:47,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:50,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:50,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:51,226 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:52,022 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:40:52,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:52,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:55,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:55,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:56,227 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:40:57,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:57,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:40:58,023 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:00,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:00,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:01,927 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:02,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:02,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:02,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:41:02,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:41:02,702 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:41:02,966 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:41:04,025 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:05,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:05,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:06,993 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:07,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:07,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:10,026 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:10,966 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:41:10,986 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:10,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:12,387 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:13,961 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:14,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:16,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:16,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:17,388 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:17,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:41:17,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:41:17,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:41:18,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:19,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:21,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:21,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:22,983 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:23,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:23,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:25,907 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:25,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:28,031 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:28,809 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:28,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:28,874 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:30,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:30,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:32,033 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:32,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:41:32,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:41:32,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:41:32,968 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:41:33,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:33,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:33,969 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:35,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:35,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:38,034 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:38,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:38,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:39,287 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:40,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:40,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:43,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:43,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:44,036 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:44,789 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:46,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:46,058 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:47,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:41:47,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:41:47,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:41:49,614 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:41:49,634 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:49,645 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:50,038 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:50,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:52,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:52,579 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:54,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:54,606 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:56,041 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:41:56,434 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:41:57,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:57,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:59,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:41:59,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:02,042 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:02,164 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:02,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:02,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:02,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:42:02,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:42:02,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:42:02,970 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:42:04,258 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:04,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:07,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:07,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:08,045 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:08,060 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:09,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:09,187 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:12,046 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:12,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:12,173 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:13,237 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:14,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:14,216 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:17,187 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:17,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:17,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:42:17,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:42:17,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:42:18,048 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:19,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:19,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:20,251 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:21,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:22,049 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:22,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:24,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:24,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:25,051 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:25,772 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:28,224 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:42:28,253 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:28,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:29,053 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:31,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:31,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:31,053 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:31,317 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:32,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:42:32,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:42:32,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:42:32,971 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:42:33,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:33,054 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:34,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:36,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:36,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:37,055 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:37,199 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:38,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:38,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:41,056 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:41,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:41,364 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:42,358 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:43,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:43,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:44,818 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:42:44,819 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:42:44,820 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:42:44,820 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:42:45,058 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:42:46,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:46,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:47,058 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:47,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:42:47,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:42:47,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:42:47,897 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:48,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:48,369 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:51,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:51,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:53,060 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:42:53,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:53,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:53,821 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:56,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:56,505 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:58,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:58,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:42:58,876 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:42:59,062 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:01,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:01,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:02,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:43:02,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:43:02,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:43:02,972 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:43:03,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:03,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:03,973 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:05,064 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:07,228 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:43:07,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:07,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:08,974 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:10,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:10,189 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:11,065 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:12,208 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:12,221 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:13,997 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:15,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:15,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:17,067 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:17,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:17,307 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:17,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:43:17,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:43:17,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:43:19,911 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:20,293 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:20,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:22,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:22,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:23,069 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:24,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:24,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:25,075 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:27,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:27,138 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:29,071 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:29,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:29,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:30,129 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:32,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:32,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:32,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:43:32,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:43:32,703 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:43:32,974 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:43:34,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:34,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:35,073 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:35,174 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:36,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:36,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:38,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:38,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:40,224 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:41,075 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:41,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:41,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:45,225 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:45,632 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:43:45,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:45,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:47,077 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:47,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:43:47,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:43:47,704 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:43:48,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:48,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:50,236 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:50,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:50,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:53,078 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:43:53,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:53,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:55,333 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:43:55,750 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:55,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:58,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:58,733 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:43:59,080 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:00,394 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:00,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:00,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:02,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:44:02,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:44:02,704 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:44:02,976 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:44:03,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:03,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:05,081 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:05,444 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:05,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:05,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:08,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:08,790 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:10,514 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:10,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:10,825 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:11,083 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:13,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:13,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:15,514 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:15,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:15,790 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:17,085 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:17,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:44:17,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:44:17,704 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:44:18,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:18,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:20,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:20,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:21,277 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:23,086 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:24,392 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:44:24,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:24,430 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:26,613 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:27,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:27,187 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:29,088 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:29,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:29,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:31,672 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:32,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:32,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:32,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:44:32,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:44:32,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:44:32,989 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:44:34,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:34,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:35,090 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:36,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:37,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:37,230 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:39,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:39,258 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:41,091 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:41,768 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:42,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:42,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:44,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:44,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:46,843 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:47,093 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:47,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:47,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:47,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:44:47,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:44:47,704 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:44:49,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:49,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:51,893 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:52,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:52,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:53,095 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:54,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:54,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:56,907 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:44:57,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:57,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:59,096 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:44:59,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:44:59,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:01,965 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:02,703 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:45:02,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:45:02,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:45:02,881 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:45:02,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:02,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:02,979 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:45:05,098 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:05,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:05,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:07,042 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:07,821 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:07,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:10,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:10,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:11,100 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:12,104 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:12,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:12,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:15,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:15,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:16,251 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:45:16,252 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:45:16,253 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:45:16,253 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:45:17,102 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:45:17,102 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:17,254 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:17,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:17,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:45:17,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:45:17,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:45:17,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:20,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:20,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:22,262 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:22,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:22,635 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:23,104 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:25,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:25,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:27,264 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:27,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:27,658 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:29,105 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:30,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:30,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:32,312 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:32,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:32,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:32,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:45:32,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:45:32,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:45:32,981 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:45:35,107 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:35,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:35,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:37,382 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:37,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:37,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:39,108 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:41,135 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:45:41,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:41,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:42,655 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:43,110 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:44,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:44,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:45,110 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:46,081 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:46,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:47,111 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:47,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:45:47,704 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:45:47,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:45:47,850 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:49,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:49,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:51,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:51,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:53,112 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:53,675 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:54,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:54,208 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:56,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:56,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:57,114 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:45:58,827 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:45:59,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:45:59,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:01,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:01,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:02,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:46:02,704 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:46:02,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:46:02,983 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:46:03,115 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:03,984 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:04,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:04,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:06,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:06,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:09,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:09,117 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:09,121 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:09,946 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:11,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:11,153 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:14,107 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:14,118 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:14,990 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:15,119 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:16,139 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:16,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:17,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:46:17,704 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:46:17,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:46:19,698 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:46:19,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:19,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:20,032 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:21,121 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:22,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:22,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:24,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:24,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:25,033 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:27,124 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:27,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:27,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:29,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:29,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:30,115 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:32,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:32,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:32,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:46:32,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:46:32,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:46:32,985 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:46:33,125 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:34,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:34,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:35,986 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:37,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:37,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:39,127 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:39,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:39,541 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:41,351 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:42,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:42,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:44,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:44,595 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:47,130 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:47,279 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:47,561 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:47,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:47,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:46:47,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:46:47,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:46:49,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:49,604 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:52,538 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:52,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:52,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:54,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:54,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:55,132 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:46:57,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:46:58,358 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:46:58,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:46:58,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:01,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:01,134 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:01,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:02,572 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:02,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:47:02,810 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:47:02,811 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:47:03,119 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:47:03,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:03,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:06,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:06,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:07,135 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:07,606 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:08,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:08,225 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:11,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:11,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:12,651 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:13,137 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:13,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:13,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:16,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:16,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:17,138 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:17,702 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:17,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:47:17,704 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:47:17,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:47:18,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:18,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:21,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:21,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:23,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:23,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:23,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:25,140 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:26,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:26,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:28,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:28,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:28,809 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:31,141 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:31,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:31,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:32,704 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:47:32,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:47:32,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:47:32,988 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:47:33,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:33,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:33,990 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:36,952 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:47:36,956 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:47:36,958 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:47:36,958 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:47:36,958 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:47:36,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:36,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:37,143 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:47:37,143 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:39,144 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:39,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:39,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:39,959 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:41,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:41,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:44,810 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:44,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:45,146 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:45,213 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:46,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:46,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:47,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:47:47,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:47:47,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:47:49,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:49,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:50,746 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:51,147 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:51,894 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:51,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:54,882 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:54,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:55,746 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:47:56,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:56,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:59,150 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:47:59,876 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:47:59,886 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:01,301 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:01,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:01,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:02,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:48:02,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:48:02,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:48:02,990 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:48:04,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:04,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:06,152 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:06,588 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:06,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:06,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:09,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:09,801 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:11,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:11,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:12,216 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:14,155 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:15,370 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:48:15,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:15,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:17,217 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:17,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:48:17,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:48:17,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:48:18,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:18,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:20,157 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:20,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:20,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:22,841 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:23,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:23,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:25,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:25,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:26,158 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:27,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:28,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:28,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:30,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:30,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:32,160 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:32,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:32,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:32,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:48:32,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:48:32,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:48:32,992 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:48:33,993 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:35,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:35,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:37,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:37,440 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:38,162 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:39,828 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:40,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:40,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:42,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:42,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:44,164 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:44,875 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:45,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:45,684 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:47,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:48:47,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:47,705 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:48:47,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:48:47,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:49,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:49,755 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:49,922 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:50,166 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:53,215 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:48:53,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:53,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:55,106 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:48:56,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:56,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:56,168 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:48:58,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:48:58,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:00,106 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:01,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:01,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:02,170 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:02,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:49:02,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:49:02,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:49:02,994 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:49:03,137 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:03,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:05,996 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:06,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:06,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:08,172 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:08,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:08,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:11,210 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:11,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:11,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:13,291 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:13,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:14,174 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:16,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:16,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:16,251 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:17,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:49:17,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:49:17,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:49:18,230 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:18,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:20,176 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:20,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:20,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:21,305 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:22,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:22,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:25,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:25,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:26,183 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:26,427 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:28,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:28,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:31,474 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:31,769 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:49:31,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:31,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:32,185 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:32,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:49:32,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:49:32,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:49:33,018 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:49:34,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:34,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:36,526 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:36,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:36,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:38,187 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:39,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:39,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:41,527 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:41,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:41,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:44,189 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:44,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:44,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:46,578 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:46,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:46,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:47,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:49:47,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:49:47,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:49:49,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:49,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:50,191 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:51,622 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:51,684 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:51,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:53,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:53,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:56,193 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:49:56,672 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:49:56,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:56,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:58,792 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:49:58,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:01,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:01,947 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:01,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:02,195 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:02,705 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:50:02,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:50:02,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:50:02,998 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:50:04,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:04,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:06,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:06,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:06,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:08,197 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:10,475 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:50:10,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:10,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:11,761 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:12,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:12,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:12,879 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:50:12,880 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:50:12,881 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:50:12,881 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:50:13,198 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:50:14,199 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:15,199 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:15,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:15,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:16,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:17,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:17,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:17,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:50:17,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:50:17,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:50:20,183 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:20,201 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:20,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:22,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:22,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:22,847 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:25,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:25,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:26,203 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:27,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:27,391 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:27,902 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:30,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:30,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:32,205 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:32,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:32,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:32,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:50:32,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:50:32,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:50:32,938 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:32,999 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:50:34,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:34,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:37,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:37,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:37,980 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:38,207 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:39,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:39,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:42,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:42,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:42,981 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:44,209 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:44,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:44,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:47,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:50:47,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:50:47,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:50:47,941 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:50:47,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:47,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:48,830 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:50,211 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:50,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:50,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:52,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:52,860 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:54,077 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:50:55,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:56,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:56,214 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:50:58,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:58,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:50:59,117 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:00,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:00,076 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:02,215 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:02,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:51:02,706 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:51:02,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:51:02,947 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:02,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:03,001 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:51:04,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:05,002 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:05,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:07,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:07,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:08,217 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:09,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:09,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:10,200 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:12,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:12,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:14,219 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:14,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:14,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:15,216 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:17,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:51:17,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:51:17,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:51:17,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:17,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:19,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:19,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:20,221 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:20,260 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:23,011 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:23,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:25,261 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:26,223 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:26,764 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:51:26,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:26,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:29,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:29,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:30,286 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:31,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:31,838 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:32,225 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:32,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:51:32,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:51:32,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:51:33,002 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:51:34,882 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:34,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:36,004 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:36,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:36,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:38,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:38,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:40,228 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:41,366 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:41,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:41,777 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:43,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:43,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:45,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:46,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:46,230 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:46,418 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:47,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:51:47,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:51:47,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:51:49,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:49,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:51,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:51,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:52,232 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:52,331 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:54,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:54,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:56,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:56,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:57,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:51:58,234 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:51:59,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:51:59,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:01,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:01,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:02,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:02,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:52:02,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:52:02,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:52:03,004 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:52:04,236 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:05,125 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:52:05,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:05,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:07,817 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:07,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:07,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:08,238 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:10,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:10,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:10,239 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:12,239 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:12,888 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:12,907 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:13,264 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:14,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:14,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:16,241 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:17,706 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:52:17,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:52:17,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:52:17,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:17,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:18,920 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:20,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:20,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:20,242 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:22,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:23,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:24,072 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:24,246 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:25,021 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:25,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:28,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:28,153 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:29,073 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:29,196 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:52:29,198 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:52:29,198 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:52:29,199 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:52:29,248 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:52:30,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:30,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:30,248 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:31,248 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:32,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:52:32,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:52:32,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:52:33,005 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:52:33,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:33,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:35,007 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:35,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:35,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:36,250 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:38,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:38,167 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:40,160 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:40,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:40,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:42,253 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:43,626 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:52:43,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:43,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:45,224 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:46,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:46,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:47,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:52:47,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:52:47,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:52:48,255 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:48,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:48,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:50,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:50,458 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:50,906 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:53,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:53,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:54,257 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:52:55,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:55,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:56,312 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:52:58,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:52:58,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:00,260 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:00,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:00,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:01,369 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:02,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:53:02,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:53:02,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:53:03,007 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:53:03,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:03,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:05,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:05,838 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:06,262 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:06,462 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:07,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:07,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:09,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:09,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:11,463 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:12,947 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:12,968 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:14,265 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:14,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:15,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:16,477 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:17,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:53:17,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:53:17,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:53:18,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:18,085 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:20,267 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:21,824 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:53:21,845 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:21,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:21,929 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:24,790 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:24,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:26,269 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:26,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:26,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:27,623 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:29,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:29,789 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:31,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:31,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:32,272 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:32,669 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:32,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:53:32,707 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:53:32,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:53:33,009 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:53:34,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:34,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:36,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:36,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:37,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:38,274 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:39,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:39,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:41,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:41,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:42,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:44,277 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:44,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:44,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:46,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:46,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:47,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:53:47,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:53:47,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:53:49,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:49,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:51,514 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:53:51,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:51,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:52,280 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:54,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:54,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:56,281 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:53:56,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:56,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:53:56,850 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:00,313 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:54:00,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:00,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:01,905 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:02,284 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:02,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:54:02,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:54:02,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:54:03,011 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:54:03,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:03,179 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:05,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:05,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:06,954 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:08,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:08,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:08,286 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:10,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:10,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:11,955 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:13,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:13,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:14,288 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:15,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:15,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:16,979 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:17,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:54:17,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:54:17,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:54:18,219 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:18,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:20,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:20,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:20,291 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:22,010 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:23,158 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:23,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:25,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:25,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:26,293 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:27,066 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:28,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:28,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:30,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:30,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:32,099 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:32,295 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:32,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:54:32,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:54:32,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:54:33,012 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:54:33,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:33,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:35,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:35,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:37,139 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:38,297 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:38,842 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:54:38,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:38,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:41,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:41,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:42,140 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:43,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:43,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:44,299 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:46,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:46,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:47,190 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:47,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:54:47,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:54:47,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:54:48,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:48,770 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:50,301 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:51,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:51,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:52,230 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:53,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:53,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:56,303 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:54:56,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:56,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:57,262 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:54:58,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:54:58,757 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:00,379 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:55:00,380 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:55:00,380 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:55:00,381 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:55:01,305 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:55:01,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:01,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:02,306 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:02,382 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:02,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:55:02,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:55:02,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:55:03,071 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:55:03,777 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:03,789 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:06,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:06,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:08,308 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:08,347 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:08,789 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:08,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:11,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:11,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:13,376 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:13,921 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:13,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:14,310 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:17,535 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:55:17,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:17,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:17,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:55:17,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:55:17,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:55:18,425 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:20,312 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:20,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:20,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:22,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:22,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:23,426 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:25,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:25,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:26,314 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:27,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:27,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:28,455 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:30,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:30,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:32,317 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:32,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:32,696 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:32,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:55:32,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:55:32,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:55:33,015 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:55:34,016 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:35,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:35,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:37,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:37,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:38,319 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:39,520 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:40,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:40,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:42,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:42,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:44,321 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:44,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:45,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:45,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:47,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:47,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:47,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:55:47,708 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:55:47,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:55:50,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:50,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:51,513 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:52,324 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:52,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:52,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:56,272 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:55:56,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:56,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:56,326 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:55:56,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:55:59,008 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:55:59,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:01,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:01,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:01,760 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:02,328 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:02,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:56:02,709 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:56:02,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:56:03,017 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:56:03,935 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:03,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:06,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:06,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:06,805 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:08,116 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:08,138 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:08,330 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:10,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:10,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:11,806 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:12,979 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:12,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:14,333 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:15,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:15,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:16,840 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:17,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:56:17,709 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:56:17,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:56:17,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:18,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:20,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:20,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:22,336 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:22,805 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:22,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:22,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:24,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:24,955 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:26,337 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:27,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:27,927 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:27,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:29,977 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:29,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:32,340 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:32,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:56:32,709 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:56:32,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:56:33,018 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:56:33,019 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:33,481 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:56:33,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:33,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:36,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:36,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:38,111 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:38,343 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:38,391 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:38,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:41,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:41,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:43,112 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:43,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:43,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:44,345 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:46,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:46,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:47,708 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:56:47,709 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:56:47,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:56:48,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:48,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:48,848 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:50,347 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:51,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:51,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:53,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:53,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:54,180 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:56:56,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:56,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:56,349 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:56:58,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:58,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:56:59,215 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:01,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:01,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:02,352 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:02,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:57:02,709 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:57:02,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:57:03,020 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:57:03,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:03,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:05,021 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:06,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:06,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:08,354 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:08,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:08,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:10,280 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:11,876 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:57:11,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:11,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:14,356 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:14,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:14,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:15,308 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:16,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:16,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:17,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:57:17,709 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:57:17,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:57:18,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:18,710 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:21,160 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:21,653 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:21,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:22,359 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:23,692 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:23,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:25,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:25,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:26,361 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:26,374 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:28,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:28,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:30,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:30,496 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:31,375 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:31,496 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:57:31,498 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:57:31,498 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:57:31,498 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:57:32,363 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:57:32,363 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:32,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:57:32,709 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:57:32,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:57:33,022 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:57:33,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:33,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:34,364 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:35,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:35,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:37,025 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:38,365 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:38,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:38,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:40,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:40,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:42,523 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:42,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:42,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:44,367 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:45,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:45,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:47,560 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:47,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:57:47,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:57:47,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:57:49,292 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:57:49,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:49,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:52,370 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:52,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:52,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:53,033 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:54,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:54,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:56,372 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:57:56,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:56,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:58,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:57:59,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:57:59,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:01,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:01,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:02,374 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:02,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:58:02,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:58:02,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:58:03,023 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:58:04,024 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:04,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:04,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:06,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:06,369 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:08,377 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:09,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:09,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:09,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:11,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:11,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:14,379 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:14,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:14,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:14,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:16,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:16,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:17,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:58:17,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:58:17,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:58:19,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:19,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:20,021 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:20,381 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:22,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:22,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:24,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:24,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:25,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:26,383 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:28,384 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:28,562 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:58:28,580 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:28,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:31,179 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:31,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:31,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:32,385 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:32,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:58:32,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:58:32,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:58:33,025 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:58:33,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:33,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:34,386 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:36,387 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:36,675 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:36,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:36,750 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:38,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:38,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:41,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:41,955 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:42,078 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:42,388 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:43,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:43,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:46,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:46,911 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:47,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:58:47,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:58:47,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:58:48,390 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:48,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:48,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:50,701 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:51,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:51,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:53,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:54,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:54,393 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:58:55,991 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:58:56,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:57,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:59,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:58:59,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:00,394 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:01,042 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:01,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:01,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:02,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:59:02,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:59:02,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:59:03,027 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:59:03,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:03,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:06,226 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:06,396 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:07,426 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:59:07,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:07,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:10,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:10,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:11,226 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:12,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:12,374 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:12,398 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:15,511 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:15,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:16,259 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:17,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:17,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:17,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:59:17,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:59:17,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:59:18,400 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:20,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:20,557 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:21,931 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:22,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:22,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:24,401 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:25,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:25,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:27,364 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:27,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:27,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:29,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:29,541 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:30,403 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:32,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:32,390 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:32,411 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:32,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:59:32,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:59:32,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:59:33,028 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 16:59:34,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:34,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:36,405 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:37,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:37,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:37,430 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:39,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:39,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:42,407 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:42,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:42,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:42,457 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:46,172 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 16:59:46,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:46,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:47,458 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:47,570 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 16:59:47,572 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 16:59:47,572 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 16:59:47,573 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 16:59:47,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 16:59:47,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 16:59:47,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 16:59:48,409 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 16:59:48,409 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:49,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:49,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:50,409 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:51,272 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:51,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:52,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 16:59:54,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:54,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:54,410 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 16:59:56,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:56,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:58,258 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:58,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 16:59:58,544 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:00,412 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:01,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:01,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:02,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:00:02,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:00:02,711 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:00:03,054 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:00:03,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:03,221 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:04,055 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:06,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:06,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:06,414 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:08,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:08,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:09,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:11,179 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:11,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:12,415 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:13,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:13,222 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:14,643 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:16,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:16,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:17,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:00:17,710 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:00:17,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:00:18,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:18,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:18,417 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:19,864 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:21,116 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:21,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:24,419 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:24,853 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:00:24,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:24,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:25,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:26,918 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:27,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:30,219 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:30,258 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:30,421 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:30,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:32,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:32,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:32,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:00:32,711 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:00:32,712 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:00:33,032 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:00:35,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:35,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:35,816 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:36,423 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:37,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:37,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:40,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:40,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:40,817 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:42,424 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:42,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:42,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:45,458 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:45,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:45,853 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:47,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:47,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:47,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:00:47,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:00:47,711 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:00:48,426 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:50,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:50,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:50,917 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:52,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:52,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:54,428 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:00:55,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:55,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:56,903 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:00:57,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:00:57,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:00,431 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:00,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:00,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:01,933 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:02,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:01:02,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:01:02,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:01:03,034 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:01:04,270 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:01:04,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:04,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:06,433 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:06,964 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:07,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:07,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:09,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:09,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:11,965 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:12,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:12,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:12,435 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:14,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:14,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:16,989 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:17,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:17,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:17,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:01:17,714 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:01:17,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:01:19,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:19,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:22,413 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:22,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:22,438 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:22,443 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:24,438 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:24,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:24,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:27,330 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:27,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:28,040 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:29,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:29,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:32,370 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:32,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:32,441 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:32,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:01:32,714 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:01:32,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:01:33,035 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:01:34,037 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:34,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:34,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:36,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:36,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:38,443 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:39,094 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:39,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:39,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:42,741 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:01:42,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:42,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:44,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:44,445 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:45,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:45,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:47,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:01:47,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:01:47,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:01:47,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:47,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:50,574 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:50,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:50,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:52,448 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:52,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:52,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:55,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:55,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:56,186 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:01:56,449 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:01:57,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:01:57,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:00,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:00,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:01,215 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:02,451 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:02:02,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:02,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:02:02,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:02:02,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:02:02,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:03,037 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:02:05,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:05,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:06,268 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:07,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:07,638 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:08,453 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:02:10,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:10,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:11,269 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:12,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:12,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:14,455 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:02:15,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:15,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:16,350 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:17,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:02:17,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:02:17,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:02:17,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:18,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:18,442 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:02:20,803 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:02:20,804 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:02:20,804 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:02:21,457 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:02:21,535 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:02:21,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:21,579 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:21,805 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:22,458 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:02:24,417 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:24,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:26,459 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:02:26,539 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:27,417 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:27,688 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:29,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:29,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:32,440 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:32,461 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:02:32,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:02:32,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:02:32,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:02:32,735 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:32,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:33,125 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:02:34,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:34,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:37,476 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:37,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:37,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:39,463 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:02:39,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:39,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:42,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:42,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:43,318 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:43,465 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:02:44,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:44,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:47,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:02:47,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:02:47,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:02:47,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:47,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:48,868 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:49,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:49,934 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:52,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:52,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:53,869 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:02:54,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:54,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:57,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:57,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:02:59,337 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:01,242 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:03:01,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:01,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:01,470 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:02,714 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:03:02,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:03:02,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:03:03,040 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:03:03,471 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:04,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:04,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:05,041 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:06,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:06,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:09,473 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:09,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:09,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:10,337 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:11,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:11,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:14,411 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:14,428 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:15,370 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:15,475 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:16,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:16,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:17,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:03:17,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:03:17,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:03:18,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:18,539 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:19,476 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:20,396 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:21,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:21,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:23,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:23,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:25,478 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:25,481 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:26,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:26,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:28,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:28,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:30,482 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:31,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:31,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:31,480 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:32,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:03:32,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:03:32,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:03:33,041 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:03:33,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:33,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:36,042 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:36,467 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:36,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:37,482 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:40,094 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:03:40,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:40,138 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:41,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:43,040 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:43,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:43,483 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:45,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:46,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:46,662 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:47,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:03:47,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:03:47,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:03:48,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:48,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:50,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:50,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:51,486 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:51,711 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:52,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:52,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:55,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:55,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:57,020 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:03:57,488 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:03:57,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:03:57,777 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:00,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:00,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:02,311 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:02,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:04:02,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:04:02,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:04:02,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:02,801 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:03,043 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:04:03,490 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:05,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:05,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:07,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:07,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:07,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:09,492 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:10,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:10,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:12,618 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:12,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:12,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:15,494 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:15,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:15,887 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:17,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:04:17,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:04:17,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:04:17,868 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:19,723 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:04:19,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:19,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:21,496 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:22,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:22,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:23,767 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:24,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:24,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:27,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:27,497 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:27,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:28,797 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:29,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:29,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:32,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:32,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:32,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:04:32,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:04:32,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:04:33,045 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:04:33,499 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:34,046 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:34,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:34,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:37,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:37,635 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:39,501 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:39,684 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:39,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:39,873 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:42,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:42,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:44,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:44,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:44,920 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:45,503 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:47,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:47,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:47,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:04:47,715 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:04:47,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:04:49,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:49,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:50,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:51,111 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:04:51,112 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:04:51,112 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:04:51,113 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:04:51,505 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:04:51,505 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:52,505 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:52,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:52,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:53,506 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:54,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:54,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:55,507 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:55,835 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:04:57,507 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:04:58,244 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:04:58,291 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:04:58,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:01,173 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:01,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:01,228 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:01,509 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:02,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:05:02,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:05:02,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:05:03,047 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:05:03,222 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:03,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:06,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:06,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:06,668 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:07,511 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:08,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:08,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:11,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:11,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:11,670 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:13,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:13,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:13,513 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:16,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:16,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:16,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:17,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:05:17,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:05:17,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:05:18,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:18,417 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:19,515 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:21,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:21,328 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:21,898 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:23,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:23,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:25,517 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:26,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:26,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:27,818 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:28,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:28,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:31,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:31,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:31,519 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:32,715 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:05:32,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:05:32,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:05:32,867 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:33,049 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:05:33,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:33,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:36,800 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:05:37,522 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:37,876 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:37,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:37,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:40,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:40,052 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:42,918 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:42,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:42,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:43,524 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:45,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:45,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:47,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:05:47,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:05:47,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:05:47,936 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:47,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:47,975 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:49,526 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:50,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:50,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:52,956 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:52,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:53,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:55,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:55,043 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:55,528 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:05:57,994 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:05:58,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:05:58,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:00,085 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:00,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:01,530 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:02,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:06:02,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:06:02,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:06:03,050 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:06:03,053 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:03,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:03,110 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:05,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:05,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:07,532 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:08,141 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:08,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:08,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:10,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:10,374 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:13,180 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:13,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:13,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:13,534 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:16,967 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:06:16,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:17,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:17,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:06:17,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:06:17,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:06:18,232 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:19,536 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:20,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:20,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:22,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:22,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:23,233 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:24,921 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:24,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:25,538 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:26,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:26,986 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:28,259 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:29,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:29,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:31,540 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:31,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:31,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:32,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:06:32,716 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:06:32,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:06:33,052 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:06:34,054 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:34,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:34,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:36,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:36,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:37,542 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:39,353 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:39,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:39,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:41,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:42,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:43,544 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:44,386 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:44,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:44,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:47,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:47,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:47,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:06:47,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:06:47,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:06:49,431 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:49,546 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:50,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:50,029 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:52,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:52,076 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:54,431 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:06:55,547 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:06:55,599 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:06:55,634 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:55,653 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:58,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:58,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:06:59,472 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:00,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:00,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:01,549 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:02,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:07:02,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:07:02,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:07:03,054 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:07:03,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:03,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:05,055 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:05,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:05,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:07,551 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:08,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:08,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:10,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:10,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:10,535 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:12,645 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:07:12,647 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:07:12,647 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:07:12,647 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:07:13,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:13,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:13,553 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:07:13,554 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:14,554 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:15,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:15,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:15,648 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:17,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:07:17,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:07:17,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:07:18,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:18,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:19,555 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:20,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:20,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:21,613 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:23,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:23,440 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:25,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:25,511 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:25,561 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:26,673 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:28,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:28,445 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:30,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:30,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:31,563 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:31,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:32,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:07:32,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:07:32,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:07:33,055 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:07:33,997 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:07:34,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:34,042 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:36,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:36,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:36,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:37,565 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:38,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:38,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:41,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:41,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:41,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:43,567 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:43,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:44,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:46,791 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:47,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:47,059 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:47,716 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:07:47,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:07:47,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:07:49,088 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:49,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:49,569 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:51,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:51,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:51,827 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:54,027 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:54,043 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:55,571 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:07:56,080 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:56,090 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:56,871 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:07:58,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:07:58,148 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:00,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:00,947 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:01,573 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:01,909 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:02,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:08:02,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:08:02,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:08:02,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:02,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:03,057 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:08:05,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:05,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:06,932 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:07,575 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:07,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:07,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:11,494 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:08:11,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:11,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:11,932 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:13,578 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:14,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:14,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:16,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:16,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:16,989 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:17,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:08:17,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:08:17,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:08:19,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:19,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:19,580 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:21,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:21,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:22,031 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:24,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:24,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:26,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:26,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:27,098 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:27,583 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:29,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:29,706 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:31,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:31,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:32,147 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:32,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:08:32,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:08:32,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:08:33,059 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:08:33,585 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:33,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:33,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:36,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:36,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:37,180 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:38,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:38,633 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:39,587 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:41,584 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:41,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:42,180 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:43,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:43,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:45,589 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:46,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:46,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:47,245 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:47,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:08:47,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:08:47,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:08:50,344 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:08:50,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:50,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:51,591 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:52,687 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:53,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:53,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:55,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:55,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:57,593 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:08:57,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:08:58,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:08:58,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:00,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:00,477 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:02,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:09:02,717 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:09:02,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:09:02,863 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:03,060 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:09:03,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:03,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:03,595 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:05,634 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:05,647 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:07,688 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:08,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:08,816 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:09,597 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:10,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:10,541 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:12,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:12,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:13,816 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:15,599 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:15,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:15,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:17,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:09:17,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:09:17,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:09:17,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:17,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:18,909 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:20,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:20,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:21,601 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:22,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:22,871 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:24,902 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:25,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:25,792 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:27,603 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:29,494 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:09:29,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:29,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:29,947 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:32,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:32,557 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:32,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:09:32,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:09:32,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:09:33,062 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:09:33,606 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:34,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:34,591 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:35,063 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:37,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:37,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:39,607 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:39,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:39,629 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:41,032 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:41,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:42,471 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:44,353 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:09:44,354 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:09:44,355 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:09:44,355 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:09:44,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:44,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:44,609 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:09:44,609 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:45,609 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:46,356 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:46,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:46,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:47,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:09:47,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:09:47,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:09:49,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:49,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:51,380 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:51,477 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:51,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:51,611 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:54,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:54,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:56,381 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:09:56,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:56,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:57,613 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:09:59,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:09:59,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:01,420 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:01,581 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:01,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:02,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:10:02,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:10:02,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:10:03,063 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:10:03,615 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:04,611 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:04,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:07,065 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:08,217 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:10:08,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:08,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:09,617 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:10,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:10,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:12,524 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:13,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:13,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:15,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:15,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:15,619 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:17,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:17,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:17,561 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:17,717 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:10:17,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:10:17,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:10:20,074 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:20,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:21,621 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:22,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:22,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:22,601 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:25,211 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:25,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:27,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:27,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:27,623 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:27,659 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:30,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:30,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:32,220 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:32,230 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:32,660 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:32,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:10:32,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:10:32,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:10:33,066 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:10:33,624 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:35,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:35,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:37,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:37,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:38,067 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:39,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:39,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:39,626 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:42,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:42,440 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:43,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:45,628 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:46,158 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:10:46,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:46,195 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:47,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:10:47,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:10:47,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:10:48,883 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:49,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:49,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:51,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:51,414 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:51,630 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:53,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:53,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:54,831 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:10:56,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:56,580 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:57,631 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:10:58,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:58,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:10:59,885 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:00,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:00,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:02,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:11:02,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:11:02,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:11:03,067 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:11:03,633 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:03,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:03,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:05,068 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:05,871 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:05,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:07,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:07,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:09,635 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:10,406 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:10,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:10,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:11,636 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:12,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:13,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:13,636 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:15,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:15,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:15,865 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:17,637 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:17,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:11:17,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:11:17,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:11:18,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:18,118 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:20,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:20,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:21,638 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:21,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:23,881 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:11:23,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:23,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:25,640 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:26,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:26,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:27,056 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:28,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:28,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:29,641 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:31,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:31,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:31,642 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:32,444 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:32,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:11:32,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:11:32,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:11:33,069 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:11:33,644 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:34,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:34,266 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:36,298 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:36,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:37,645 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:37,910 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:38,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:38,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:39,646 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:41,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:41,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:43,259 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:43,417 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:43,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:45,221 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:11:45,223 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:11:45,223 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:11:45,224 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:11:45,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:45,648 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:11:45,649 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:46,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:47,650 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:47,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:11:47,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:11:47,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:11:48,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:48,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:48,867 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:50,649 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:50,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:51,651 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:53,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:53,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:54,664 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:11:55,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:55,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:57,654 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:11:57,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:11:57,779 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:00,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:01,251 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:12:01,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:01,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:02,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:12:02,719 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:12:02,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:12:03,070 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:12:03,656 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:04,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:04,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:06,071 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:06,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:06,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:08,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:08,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:09,658 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:11,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:11,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:11,915 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:13,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:13,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:15,660 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:15,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:15,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:17,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:12:17,719 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:12:17,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:12:18,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:18,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:20,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:20,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:21,138 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:21,662 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:24,214 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:24,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:25,664 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:26,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:26,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:27,021 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:28,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:28,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:30,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:30,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:31,665 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:32,458 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:32,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:12:32,718 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:12:32,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:12:33,072 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:12:33,322 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:33,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:35,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:35,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:37,667 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:38,346 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:39,113 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:12:39,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:39,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:42,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:42,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:43,669 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:44,119 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:44,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:44,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:46,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:46,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:47,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:12:47,719 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:12:47,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:12:49,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:49,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:49,671 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:49,957 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:51,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:51,574 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:54,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:54,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:55,557 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:12:55,674 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:12:56,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:56,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:58,801 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:12:58,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:00,677 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:01,004 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:01,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:01,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:02,718 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:13:02,719 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:13:02,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:13:03,074 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:13:03,971 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:03,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:06,008 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:06,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:06,679 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:06,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:09,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:09,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:11,139 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:11,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:12,433 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:12,681 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:13,209 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:13,219 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:16,744 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:13:16,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:16,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:17,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:13:17,719 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:13:17,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:13:17,890 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:18,683 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:19,748 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:19,770 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:21,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:21,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:23,141 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:23,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:23,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:24,686 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:25,905 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:27,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:28,825 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:29,076 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:29,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:30,689 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:31,110 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:31,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:32,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:13:32,719 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:13:32,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:13:33,075 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:13:34,132 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:34,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:34,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:36,165 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:36,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:36,691 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:38,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:38,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:40,123 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:40,693 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:41,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:41,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:43,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:43,319 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:45,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:45,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:46,050 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:46,694 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:47,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:13:47,719 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:13:47,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:13:48,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:48,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:50,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:50,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:51,912 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:52,696 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:54,049 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:13:54,077 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:54,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:57,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:57,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:57,870 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:13:58,699 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:13:59,138 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:13:59,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:02,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:02,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:02,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:14:02,719 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:14:02,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:14:02,878 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:03,077 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:14:04,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:04,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:04,701 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:06,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:06,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:08,078 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:08,309 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:14:08,311 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:14:08,311 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:14:08,311 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:14:08,702 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:14:09,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:09,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:09,702 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:10,703 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:11,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:11,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:13,318 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:13,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:13,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:16,706 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:16,745 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:16,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:17,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:14:17,720 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:14:17,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:14:18,777 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:18,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:20,779 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:20,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:20,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:22,708 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:23,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:23,825 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:26,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:26,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:26,516 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:28,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:28,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:28,710 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:32,104 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:32,121 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:14:32,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:32,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:32,711 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:32,719 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:14:32,720 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:14:32,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:14:33,079 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:14:34,179 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:34,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:37,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:37,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:38,045 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:38,714 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:39,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:39,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:41,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:42,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:43,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:44,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:44,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:44,716 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:46,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:46,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:47,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:14:47,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:14:47,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:14:49,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:49,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:50,633 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:51,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:51,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:52,718 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:53,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:53,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:56,402 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:14:56,720 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:14:57,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:57,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:59,065 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:14:59,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:01,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:01,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:01,746 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:02,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:15:02,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:15:02,722 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:15:02,724 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:03,080 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:15:04,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:04,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:06,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:06,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:07,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:08,726 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:09,717 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:15:09,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:09,755 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:12,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:12,767 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:13,335 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:14,728 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:14,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:14,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:17,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:15:17,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:15:17,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:15:17,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:17,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:18,894 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:19,759 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:19,773 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:20,730 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:22,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:22,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:24,000 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:24,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:25,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:26,731 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:29,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:29,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:29,518 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:31,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:32,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:32,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:15:32,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:15:32,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:15:32,733 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:33,138 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:15:34,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:35,140 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:35,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:38,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:38,735 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:38,826 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:40,353 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:41,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:41,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:44,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:44,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:44,737 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:45,663 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:46,183 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:46,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:47,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:15:47,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:15:47,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:15:49,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:49,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:50,740 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:51,172 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:52,993 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:15:53,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:53,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:55,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:55,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:56,180 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:15:56,741 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:15:58,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:15:58,074 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:00,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:00,130 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:01,181 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:02,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:02,207 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:02,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:16:02,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:16:02,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:16:03,084 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:16:04,745 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:05,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:05,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:07,085 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:07,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:07,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:09,179 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:09,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:10,746 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:11,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:11,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:12,411 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:14,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:14,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:16,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:16,485 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:16,748 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:17,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:16:17,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:16:17,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:16:17,918 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:18,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:18,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:21,435 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:21,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:22,750 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:22,987 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:23,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:23,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:25,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:25,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:27,988 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:29,369 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:16:29,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:29,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:30,752 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:32,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:32,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:32,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:16:32,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:16:32,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:16:33,085 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:16:33,087 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:34,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:34,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:36,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:36,595 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:36,754 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:38,750 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:39,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:39,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:40,568 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:16:40,570 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:16:40,570 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:16:40,571 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:16:40,755 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:16:41,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:41,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:41,755 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:42,756 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:43,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:43,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:44,571 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:46,806 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:46,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:47,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:16:47,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:16:47,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:16:48,758 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:48,854 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:48,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:50,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:50,926 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:51,002 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:52,759 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:16:53,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:53,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:55,977 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:55,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:56,941 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:16:58,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:58,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:16:58,761 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:01,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:01,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:02,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:17:02,721 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:17:02,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:17:02,932 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:03,086 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:17:03,165 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:03,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:04,763 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:06,897 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:17:06,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:06,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:08,438 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:09,838 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:09,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:10,765 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:11,882 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:11,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:13,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:13,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:14,320 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:16,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:16,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:16,767 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:17,721 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:17:17,722 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:17:17,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:17:19,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:19,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:20,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:21,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:21,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:22,768 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:23,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:24,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:25,788 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:26,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:26,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:28,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:28,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:28,770 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:31,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:31,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:31,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:32,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:17:32,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:17:32,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:17:33,088 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:17:33,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:33,535 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:34,773 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:35,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:35,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:37,090 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:38,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:38,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:38,774 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:40,649 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:40,676 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:42,148 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:44,448 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:17:44,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:44,492 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:44,777 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:47,557 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:47,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:47,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:47,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:17:47,722 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:17:47,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:17:49,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:49,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:50,779 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:51,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:51,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:53,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:17:54,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:54,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:56,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:56,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:56,781 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:17:58,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:58,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:17:59,485 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:01,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:01,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:02,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:18:02,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:18:02,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:18:02,782 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:03,090 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:18:03,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:04,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:05,091 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:06,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:06,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:08,785 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:09,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:09,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:10,954 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:11,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:11,160 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:13,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:13,215 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:14,786 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:16,097 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:16,116 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:16,503 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:17,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:18:17,722 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:18:17,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:18:18,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:18,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:20,788 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:21,853 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:18:21,877 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:21,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:22,444 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:24,789 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:24,977 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:25,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:27,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:27,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:27,979 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:29,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:30,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:30,791 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:32,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:32,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:32,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:18:32,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:18:32,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:18:33,091 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:18:33,093 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:34,987 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:35,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:36,793 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:37,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:37,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:38,877 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:39,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:40,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:42,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:42,058 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:42,795 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:44,444 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:44,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:44,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:46,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:46,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:47,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:18:47,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:18:47,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:18:48,797 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:49,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:49,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:49,886 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:51,900 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:51,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:54,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:54,798 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:18:54,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:54,933 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:18:56,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:18:56,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:00,636 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:19:00,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:00,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:00,800 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:00,854 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:02,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:19:02,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:19:02,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:19:03,093 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:19:03,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:03,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:04,481 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:19:04,483 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:19:04,483 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:19:04,484 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:19:04,801 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:19:05,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:05,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:05,802 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:06,484 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:06,802 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:07,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:07,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:10,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:10,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:12,352 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:12,804 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:12,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:13,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:15,029 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:15,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:16,805 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:17,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:19:17,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:19:17,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:19:17,929 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:18,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:18,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:20,095 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:20,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:22,807 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:23,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:23,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:23,925 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:25,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:25,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:27,484 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:27,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:28,809 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:29,875 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:30,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:30,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:32,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:32,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:32,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:19:32,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:19:32,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:19:33,095 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:19:34,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:34,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:34,812 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:35,250 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:38,107 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:19:38,135 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:38,156 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:40,814 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:41,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:41,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:41,190 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:43,128 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:43,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:45,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:45,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:46,795 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:46,816 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:47,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:19:47,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:19:47,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:19:48,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:48,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:50,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:50,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:52,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:52,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:52,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:52,818 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:55,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:55,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:56,819 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:19:57,401 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:57,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:58,300 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:19:59,449 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:19:59,462 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:02,494 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:02,518 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:02,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:20:02,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:20:02,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:20:02,820 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:03,097 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:20:04,098 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:04,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:04,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:07,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:07,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:08,822 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:09,127 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:09,779 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:09,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:11,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:11,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:14,699 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:14,824 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:15,391 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:20:15,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:15,445 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:17,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:20:17,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:20:17,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:20:18,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:18,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:20,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:20,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:20,625 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:20,826 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:22,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:22,637 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:25,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:25,674 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:26,423 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:26,829 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:27,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:27,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:29,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:29,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:32,361 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:32,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:32,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:20:32,723 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:20:32,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:20:32,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:32,831 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:33,099 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:20:34,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:34,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:36,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:36,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:37,939 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:39,833 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:39,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:39,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:41,912 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:41,926 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:43,801 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:43,971 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:43,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:45,835 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:46,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:46,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:47,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:20:47,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:20:47,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:20:48,937 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:48,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:49,617 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:51,838 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:52,672 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:20:52,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:52,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:55,442 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:20:55,635 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:55,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:55,839 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:20:57,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:20:57,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:00,939 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:00,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:01,386 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:01,841 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:02,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:21:02,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:21:02,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:21:03,011 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:03,021 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:03,101 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:21:05,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:05,067 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:06,946 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:07,843 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:08,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:08,129 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:10,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:10,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:12,187 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:12,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:12,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:13,845 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:15,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:15,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:17,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:17,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:17,723 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:21:17,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:21:17,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:21:17,956 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:19,846 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:20,483 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:20,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:22,584 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:22,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:22,957 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:25,757 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:25,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:25,848 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:27,848 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:27,868 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:28,337 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:28,936 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:21:28,937 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:21:28,938 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:21:28,938 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:21:29,849 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:21:29,849 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:31,599 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:21:31,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:31,633 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:31,849 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:32,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:21:32,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:21:32,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:21:33,102 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:21:34,103 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:34,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:34,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:36,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:36,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:37,851 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:39,368 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:39,852 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:39,947 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:41,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:41,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:43,854 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:44,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:44,029 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:45,803 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:47,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:47,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:47,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:21:47,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:21:47,725 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:21:47,855 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:49,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:49,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:50,858 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:51,208 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:51,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:53,858 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:21:54,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:54,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:56,177 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:21:56,238 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:56,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:58,280 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:58,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:21:59,860 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:01,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:01,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:01,500 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:02,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:22:02,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:22:02,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:22:03,103 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:22:03,285 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:03,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:05,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:05,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:05,861 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:07,446 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:08,813 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:22:08,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:08,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:11,864 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:11,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:11,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:13,020 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:14,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:14,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:16,054 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:16,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:17,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:22:17,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:22:17,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:22:17,866 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:19,097 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:19,118 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:20,903 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:21,141 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:21,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:23,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:23,208 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:23,868 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:25,903 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:26,160 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:26,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:27,869 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:28,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:28,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:30,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:30,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:31,369 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:32,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:22:32,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:22:32,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:22:33,104 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:22:33,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:33,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:33,870 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:35,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:35,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:37,106 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:37,421 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:37,439 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:39,872 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:40,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:40,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:42,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:42,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:42,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:45,753 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:22:45,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:45,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:45,874 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:47,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:22:47,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:22:47,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:22:47,908 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:48,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:48,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:50,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:50,706 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:51,877 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:53,231 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:22:53,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:53,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:55,591 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:55,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:57,879 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:22:58,521 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:58,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:22:59,119 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:00,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:00,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:02,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:23:02,724 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:23:02,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:23:03,106 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:23:03,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:03,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:03,881 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:05,107 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:05,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:05,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:07,882 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:08,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:08,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:10,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:10,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:10,610 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:13,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:13,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:13,884 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:15,360 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:15,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:16,564 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:17,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:23:17,725 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:23:17,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:23:18,306 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:18,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:19,886 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:20,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:20,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:22,089 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:23,701 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:23:23,732 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:23,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:25,888 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:26,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:26,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:28,007 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:28,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:28,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:31,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:31,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:31,890 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:32,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:23:32,725 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:23:32,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:23:33,108 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:23:33,110 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:33,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:33,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:36,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:36,580 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:37,893 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:38,408 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:38,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:38,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:41,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:41,717 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:43,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:43,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:43,896 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:44,329 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:46,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:46,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:47,724 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:23:47,725 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:23:47,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:23:47,897 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:48,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:48,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:49,966 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:51,672 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:51,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:52,602 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:23:52,603 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:23:52,603 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:23:52,604 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:23:52,899 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:23:53,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:53,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:53,899 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:23:55,605 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:23:56,511 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:56,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:58,539 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:58,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:23:59,901 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:01,356 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:01,971 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:24:01,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:02,000 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:02,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:24:02,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:24:02,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:24:03,109 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:24:04,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:04,804 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:06,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:06,847 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:07,111 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:09,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:09,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:11,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:11,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:12,112 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:14,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:14,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:16,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:16,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:17,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:24:17,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:24:17,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:24:17,907 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:17,922 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:19,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:19,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:21,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:21,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:21,909 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:23,666 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:24,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:24,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:26,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:26,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:27,910 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:29,405 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:29,406 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:29,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:31,453 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:31,466 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:32,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:24:32,725 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:24:32,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:24:33,111 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:24:33,913 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:34,401 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:34,422 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:35,113 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:36,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:36,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:39,823 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:24:39,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:39,862 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:39,917 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:40,806 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:42,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:42,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:44,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:44,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:45,919 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:46,720 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:47,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:47,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:47,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:24:47,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:24:47,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:24:49,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:49,698 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:51,921 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:52,157 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:52,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:52,693 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:54,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:54,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:57,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:57,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:57,923 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:24:58,066 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:24:59,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:24:59,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:01,925 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:02,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:02,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:02,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:25:02,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:25:02,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:25:03,113 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:25:03,115 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:04,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:04,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:07,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:07,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:07,927 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:08,521 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:09,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:09,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:12,364 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:12,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:13,930 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:14,394 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:14,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:14,453 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:17,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:25:17,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:25:17,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:25:17,960 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:25:17,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:18,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:19,861 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:19,932 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:20,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:20,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:22,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:22,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:25,623 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:25,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:25,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:25,934 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:27,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:27,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:30,688 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:30,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:31,132 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:31,937 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:32,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:32,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:25:32,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:25:32,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:25:32,738 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:33,115 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:25:35,595 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:35,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:37,024 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:37,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:37,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:37,938 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:40,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:40,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:41,939 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:42,483 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:42,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:42,657 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:45,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:45,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:47,643 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:47,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:47,725 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:25:47,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:25:47,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:25:47,940 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:50,476 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:50,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:50,965 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:52,505 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:52,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:53,943 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:25:56,004 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:25:56,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:56,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:56,334 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:25:58,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:58,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:25:59,945 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:00,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:00,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:01,857 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:02,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:26:02,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:26:02,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:26:03,117 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:26:03,721 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:03,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:05,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:05,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:05,947 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:07,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:08,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:08,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:10,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:10,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:11,950 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:13,217 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:13,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:13,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:15,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:15,845 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:17,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:26:17,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:26:17,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:26:17,952 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:18,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:18,811 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:18,886 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:20,837 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:20,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:21,953 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:23,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:23,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:24,646 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:25,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:25,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:26,635 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:26:26,637 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:26:26,637 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:26:26,638 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:26:26,955 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:26:27,955 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:28,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:28,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:30,638 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:30,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:30,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:32,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:26:32,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:26:32,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:26:33,119 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:26:33,957 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:34,060 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:26:34,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:34,100 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:36,120 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:36,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:36,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:38,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:38,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:39,959 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:40,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:40,835 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:41,997 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:43,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:43,624 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:45,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:45,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:45,962 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:47,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:26:47,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:26:47,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:26:48,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:48,615 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:50,633 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:50,646 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:50,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:51,965 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:53,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:53,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:55,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:55,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:56,292 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:26:57,966 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:26:58,298 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:26:58,311 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:00,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:00,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:01,810 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:01,968 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:02,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:27:02,727 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:27:02,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:27:03,121 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:27:03,188 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:03,207 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:05,224 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:05,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:07,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:07,970 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:08,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:08,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:11,584 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:27:11,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:11,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:13,320 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:13,972 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:14,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:14,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:16,602 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:16,611 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:17,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:27:17,726 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:27:17,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:27:19,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:19,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:19,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:19,974 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:21,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:21,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:24,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:24,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:25,029 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:25,976 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:26,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:26,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:28,326 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:28,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:30,381 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:31,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:31,085 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:31,978 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:32,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:27:32,727 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:27:32,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:27:33,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:33,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:33,123 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:27:35,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:35,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:36,266 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:37,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:37,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:37,980 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:39,992 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:40,002 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:41,805 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:41,982 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:42,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:42,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:44,877 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:44,886 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:47,707 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:47,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:27:47,727 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:27:47,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:27:47,983 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:48,198 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:27:48,221 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:48,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:50,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:50,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:52,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:52,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:53,145 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:53,986 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:27:55,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:55,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:57,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:57,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:27:58,713 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:27:59,988 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:00,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:00,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:02,707 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:02,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:02,726 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:28:02,727 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:28:02,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:28:03,124 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:28:04,589 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:05,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:05,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:05,990 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:07,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:07,559 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:09,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:10,106 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:10,642 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:11,992 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:12,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:12,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:15,497 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:15,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:15,992 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:15,993 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:17,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:17,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:17,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:28:17,727 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:28:17,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:28:20,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:20,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:21,556 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:21,995 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:22,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:22,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:25,708 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:28:25,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:25,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:27,448 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:27,997 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:28,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:28,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:30,588 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:30,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:32,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:28:32,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:28:32,728 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:28:32,946 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:33,126 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:28:33,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:33,485 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:33,999 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:35,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:35,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:38,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:38,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:38,858 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:40,000 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:40,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:40,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:43,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:43,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:44,698 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:45,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:45,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:46,002 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:47,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:28:47,727 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:28:47,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:28:48,291 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:48,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:49,323 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:28:49,324 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:28:49,325 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:28:49,325 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:28:50,003 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:28:50,004 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:50,326 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:50,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:50,339 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:52,004 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:53,148 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:53,167 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:55,191 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:55,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:56,143 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:28:57,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:28:58,005 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:28:58,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:00,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:00,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:01,670 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:02,007 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:02,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:29:02,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:29:02,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:29:03,128 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:29:03,555 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:29:03,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:03,589 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:06,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:06,366 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:07,569 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:08,009 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:08,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:08,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:11,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:11,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:13,116 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:13,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:13,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:14,011 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:16,099 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:16,119 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:17,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:29:17,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:29:17,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:29:18,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:18,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:18,957 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:20,013 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:20,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:20,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:22,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:23,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:24,412 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:25,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:25,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:26,014 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:27,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:28,013 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:30,015 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:30,120 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:30,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:30,932 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:32,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:29:32,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:29:32,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:29:32,953 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:32,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:33,130 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:29:35,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:35,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:36,007 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:36,017 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:37,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:37,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:41,078 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:29:41,100 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:41,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:41,466 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:42,019 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:43,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:43,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:45,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:45,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:47,344 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:47,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:29:47,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:29:47,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:29:48,021 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:48,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:48,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:50,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:50,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:52,812 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:29:53,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:53,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:54,023 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:29:55,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:55,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:58,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:58,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:29:58,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:00,025 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:00,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:00,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:02,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:30:02,728 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:30:02,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:30:03,131 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:30:03,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:03,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:04,310 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:05,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:05,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:06,027 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:08,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:08,315 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:10,028 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:10,162 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:10,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:10,357 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:13,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:13,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:15,358 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:15,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:16,031 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:16,037 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:17,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:30:17,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:30:17,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:30:18,800 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:30:18,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:18,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:21,058 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:21,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:21,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:22,032 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:23,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:23,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:26,059 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:26,485 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:26,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:28,034 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:28,540 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:28,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:31,350 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:31,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:31,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:32,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:30:32,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:30:32,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:30:33,133 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:30:33,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:33,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:34,037 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:36,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:36,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:37,135 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:38,397 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:38,411 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:40,040 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:41,450 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:41,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:42,785 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:43,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:43,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:44,041 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:46,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:46,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:47,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:30:47,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:30:47,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:30:47,905 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:48,607 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:48,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:50,043 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:51,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:51,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:53,045 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:53,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:53,461 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:56,045 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:30:56,734 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:30:56,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:56,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:58,895 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:30:59,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:30:59,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:01,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:01,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:02,048 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:02,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:31:02,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:31:02,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:31:03,135 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:31:04,136 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:04,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:04,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:06,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:06,618 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:08,050 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:09,323 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:09,506 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:09,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:11,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:11,563 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:11,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:31:11,741 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:31:11,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:31:11,742 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:31:12,051 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:31:12,051 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:13,051 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:14,378 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:14,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:14,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:17,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:17,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:17,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:31:17,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:31:17,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:31:18,054 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:19,905 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:20,168 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:20,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:22,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:22,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:24,056 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:25,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:25,033 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:25,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:27,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:27,080 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:29,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:29,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:30,059 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:30,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:31,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:31,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:32,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:31:32,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:31:32,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:31:33,137 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:31:35,218 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:31:35,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:35,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:36,061 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:36,690 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:37,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:37,960 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:39,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:39,989 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:42,063 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:42,497 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:42,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:42,835 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:44,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:44,871 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:47,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:31:47,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:31:47,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:31:47,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:47,757 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:48,065 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:49,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:49,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:50,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:52,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:52,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:54,067 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:54,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:54,825 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:55,890 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:31:57,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:57,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:58,069 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:31:59,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:31:59,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:01,763 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:02,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:02,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:02,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:32:02,729 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:32:02,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:32:03,138 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:32:04,071 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:04,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:04,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:07,277 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:07,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:07,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:09,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:09,702 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:10,074 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:13,105 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:32:13,127 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:13,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:13,142 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:15,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:15,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:16,076 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:17,729 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:32:17,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:32:17,730 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:32:17,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:17,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:20,766 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:20,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:20,839 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:22,078 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:22,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:22,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:25,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:25,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:25,767 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:27,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:27,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:28,080 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:30,675 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:30,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:31,223 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:32,081 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:32,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:32:32,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:32:32,731 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:32:32,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:32,746 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:33,140 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:32:35,528 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:35,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:37,141 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:37,571 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:37,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:38,084 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:40,366 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:40,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:42,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:42,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:42,594 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:44,086 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:45,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:45,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:47,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:47,421 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:47,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:32:47,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:32:47,730 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:32:47,868 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:50,088 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:50,740 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:32:50,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:50,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:52,975 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:53,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:53,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:55,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:55,465 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:56,090 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:32:58,432 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:32:58,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:32:58,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:00,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:00,518 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:02,092 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:02,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:33:02,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:33:02,731 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:33:03,142 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:33:03,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:03,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:04,143 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:05,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:05,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:06,094 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:08,094 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:08,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:08,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:09,650 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:10,095 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:10,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:10,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:13,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:13,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:15,086 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:15,097 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:15,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:15,567 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:17,097 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:17,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:33:17,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:33:17,731 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:33:18,180 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:33:18,182 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:33:18,182 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:33:18,182 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:33:18,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:18,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:19,098 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:33:19,098 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:20,183 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:20,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:21,098 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:21,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:23,099 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:25,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:25,268 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:25,314 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:27,101 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:27,287 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:27,296 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:30,315 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:30,896 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:33:30,920 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:30,933 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:32,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:33:32,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:33:32,731 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:33:33,102 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:33,144 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:33:33,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:33,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:35,845 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:35,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:36,146 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:38,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:38,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:39,105 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:40,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:40,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:41,350 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:43,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:43,804 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:45,107 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:45,821 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:45,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:46,400 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:47,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:33:47,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:33:47,731 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:33:48,825 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:48,845 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:50,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:50,884 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:51,108 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:51,422 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:52,905 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:52,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:55,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:55,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:56,493 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:33:57,110 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:33:57,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:33:57,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:00,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:00,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:01,493 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:02,699 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:02,713 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:02,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:34:02,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:34:02,731 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:34:03,112 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:03,145 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:34:05,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:05,712 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:07,147 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:09,115 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:09,320 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:34:09,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:09,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:11,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:11,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:12,581 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:14,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:14,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:15,117 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:16,197 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:16,208 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:17,600 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:17,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:34:17,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:34:17,731 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:34:19,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:19,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:21,070 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:21,080 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:21,119 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:22,642 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:23,098 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:23,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:25,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:25,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:27,121 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:27,661 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:27,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:27,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:30,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:30,923 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:32,710 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:32,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:34:32,731 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:34:32,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:34:32,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:32,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:33,123 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:33,147 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:34:35,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:35,975 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:37,748 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:37,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:38,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:39,125 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:40,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:41,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:42,748 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:43,093 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:43,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:45,127 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:46,718 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:34:46,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:46,783 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:47,730 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:34:47,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:34:47,732 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:34:47,890 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:48,804 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:48,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:51,129 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:51,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:51,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:53,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:53,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:53,871 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:34:55,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:55,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:57,131 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:34:58,593 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:58,608 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:34:58,908 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:00,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:00,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:02,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:02,669 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:02,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:35:02,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:35:02,732 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:35:03,133 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:03,148 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:35:04,150 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:05,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:05,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:07,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:07,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:09,135 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:09,969 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:10,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:10,512 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:12,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:12,549 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:14,990 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:15,137 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:15,478 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:15,489 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:17,509 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:17,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:17,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:35:17,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:35:17,732 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:35:20,176 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:20,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:20,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:21,139 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:24,231 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:35:24,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:24,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:25,177 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:27,142 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:27,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:27,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:29,369 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:29,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:30,232 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:32,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:32,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:32,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:35:32,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:35:32,732 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:35:33,144 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:33,150 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:35:34,377 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:34,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:36,151 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:37,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:37,393 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:39,146 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:39,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:39,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:41,280 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:42,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:42,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:44,445 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:35:44,447 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:35:44,447 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:35:44,448 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:35:44,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:44,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:45,148 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:35:45,149 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:46,448 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:47,149 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:47,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:47,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:47,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:35:47,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:35:47,732 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:35:49,670 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:49,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:51,150 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:51,456 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:51,727 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:51,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:54,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:54,708 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:56,457 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:35:56,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:56,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:57,152 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:35:59,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:35:59,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:01,490 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:02,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:36:02,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:36:02,732 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:36:03,151 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:36:03,155 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:03,337 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:36:03,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:03,365 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:06,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:06,467 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:07,153 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:08,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:08,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:09,157 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:11,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:11,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:12,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:13,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:13,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:15,159 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:16,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:16,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:17,599 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:17,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:36:17,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:36:17,732 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:36:18,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:18,580 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:21,161 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:21,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:21,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:22,633 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:23,570 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:23,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:26,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:26,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:27,656 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:28,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:28,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:29,164 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:31,747 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:31,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:32,657 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:32,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:36:32,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:36:32,733 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:36:33,154 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:36:33,792 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:33,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:35,166 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:36,691 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:36,714 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:38,156 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:38,759 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:38,771 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:41,168 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:42,380 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:36:42,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:42,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:43,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:45,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:45,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:47,170 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:47,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:47,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:47,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:36:47,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:36:47,733 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:36:50,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:50,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:50,396 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:52,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:52,286 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:53,172 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:36:54,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:54,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:55,816 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:36:57,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:57,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:36:59,174 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:00,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:00,112 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:00,915 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:02,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:37:02,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:37:02,733 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:37:03,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:03,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:03,155 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:37:05,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:05,090 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:05,176 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:06,156 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:08,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:08,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:10,252 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:10,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:11,178 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:12,015 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:13,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:13,181 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:15,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:15,225 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:17,044 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:17,180 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:17,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:37:17,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:37:17,733 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:37:18,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:18,197 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:21,790 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:37:21,824 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:21,834 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:22,248 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:23,182 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:24,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:24,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:26,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:26,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:27,249 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:29,184 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:29,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:29,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:31,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:31,959 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:32,288 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:32,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:37:32,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:37:32,733 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:37:33,157 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:37:34,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:34,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:35,186 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:36,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:37,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:37,327 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:39,884 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:39,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:41,188 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:41,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:41,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:42,367 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:44,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:44,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:46,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:46,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:47,190 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:47,413 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:47,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:37:47,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:37:47,733 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:37:50,031 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:50,060 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:52,095 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:52,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:52,448 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:37:53,192 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:37:55,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:55,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:57,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:57,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:37:57,449 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:00,195 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:00,714 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:38:00,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:00,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:02,477 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:02,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:38:02,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:38:02,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:38:03,158 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:38:03,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:03,703 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:05,744 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:05,763 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:06,197 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:07,522 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:07,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:07,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:10,740 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:10,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:12,199 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:12,562 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:12,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:12,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:15,674 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:38:15,676 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:38:15,676 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:38:15,677 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:38:15,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:15,867 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:16,200 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:38:17,200 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:17,677 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:17,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:38:17,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:38:17,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:38:17,888 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:17,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:18,200 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:20,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:20,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:22,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:22,907 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:23,658 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:24,202 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:25,795 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:25,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:27,854 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:27,865 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:28,709 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:29,901 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:30,204 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:30,955 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:32,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:38:32,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:38:32,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:38:32,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:32,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:33,160 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:38:33,753 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:36,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:36,137 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:36,206 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:38,754 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:39,756 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:38:39,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:39,801 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:42,208 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:42,681 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:42,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:43,765 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:44,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:44,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:46,755 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:46,767 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:47,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:38:47,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:38:47,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:38:48,210 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:48,902 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:49,518 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:49,533 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:51,558 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:51,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:54,211 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:38:54,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:54,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:54,828 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:38:56,502 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:56,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:59,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:59,498 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:38:59,872 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:00,214 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:01,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:01,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:02,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:39:02,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:39:02,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:39:03,162 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:39:04,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:04,532 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:04,931 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:06,215 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:06,569 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:06,594 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:09,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:09,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:09,932 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:11,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:11,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:12,217 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:14,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:14,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:14,970 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:17,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:39:17,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:39:17,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:39:18,219 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:18,377 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:39:18,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:18,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:20,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:20,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:20,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:23,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:23,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:24,221 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:25,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:25,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:26,386 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:27,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:27,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:28,222 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:30,222 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:30,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:30,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:31,842 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:32,223 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:32,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:32,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:32,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:39:32,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:39:32,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:39:33,163 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:39:35,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:35,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:36,224 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:37,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:37,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:37,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:40,225 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:40,446 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:40,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:42,480 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:42,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:43,394 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:45,621 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:45,634 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:46,227 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:47,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:39:47,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:39:47,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:39:47,948 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:47,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:48,976 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:50,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:50,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:52,229 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:52,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:52,424 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:54,955 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:39:56,065 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:39:56,086 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:56,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:58,231 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:39:58,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:39:59,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:00,001 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:01,041 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:01,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:02,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:40:02,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:40:02,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:40:03,165 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:40:03,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:03,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:04,233 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:05,167 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:05,953 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:05,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:08,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:08,960 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:10,238 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:10,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:11,009 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:11,087 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:14,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:14,163 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:16,128 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:16,208 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:16,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:16,242 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:17,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:40:17,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:40:17,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:40:19,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:19,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:21,154 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:21,171 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:21,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:22,243 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:24,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:24,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:26,154 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:26,190 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:26,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:28,245 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:29,271 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:29,295 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:31,197 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:31,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:31,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:32,297 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:40:32,298 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:40:32,299 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:40:32,299 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:40:32,733 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:40:32,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:40:32,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:40:33,167 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:40:33,248 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:40:33,248 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:34,248 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:34,901 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:40:34,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:34,953 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:37,168 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:37,669 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:37,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:39,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:39,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:40,249 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:42,273 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:42,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:42,716 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:44,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:44,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:46,251 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:47,335 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:47,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:40:47,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:40:47,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:40:47,814 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:47,832 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:49,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:49,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:52,253 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:40:52,353 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:52,864 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:52,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:54,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:54,931 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:57,393 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:40:57,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:57,977 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:40:58,256 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:00,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:00,025 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:02,393 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:02,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:41:02,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:41:02,734 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:41:02,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:03,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:03,169 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:41:04,258 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:05,041 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:05,058 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:08,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:08,040 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:08,171 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:10,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:10,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:10,260 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:13,475 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:13,622 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:41:13,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:13,688 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:16,262 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:16,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:16,426 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:17,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:41:17,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:41:17,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:41:18,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:18,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:18,913 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:21,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:21,341 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:22,264 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:23,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:23,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:24,567 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:25,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:25,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:28,265 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:28,349 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:28,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:29,597 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:30,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:30,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:32,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:41:32,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:41:32,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:41:33,171 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:41:33,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:33,307 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:34,267 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:35,172 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:35,345 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:35,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:38,186 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:38,205 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:40,241 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:40,251 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:40,269 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:40,692 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:43,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:43,259 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:45,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:45,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:45,722 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:46,271 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:47,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:41:47,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:41:47,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:41:48,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:48,301 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:51,236 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:51,925 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:41:51,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:51,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:54,274 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:41:55,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:55,065 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:56,237 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:41:57,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:57,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:41:58,275 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:00,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:00,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:01,805 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:02,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:02,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:02,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:42:02,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:42:02,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:42:03,172 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:42:04,277 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:04,308 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:04,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:06,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:06,384 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:07,174 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:08,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:08,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:10,278 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:11,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:11,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:12,893 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:13,392 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:13,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:16,263 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:16,280 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:16,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:17,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:42:17,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:42:17,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:42:18,324 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:18,340 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:18,873 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:20,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:20,424 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:22,282 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:22,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:22,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:23,954 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:25,226 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:25,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:28,284 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:28,993 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:29,086 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:42:29,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:29,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:31,164 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:31,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:32,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:42:32,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:42:32,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:42:33,173 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:42:34,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:34,175 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:34,200 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:34,285 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:36,248 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:36,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:38,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:38,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:39,175 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:41,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:41,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:42,288 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:43,235 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:43,260 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:44,276 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:46,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:46,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:47,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:42:47,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:42:47,736 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:42:48,290 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:48,505 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:48,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:49,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:51,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:51,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:53,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:53,833 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:54,295 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:42:55,360 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:42:55,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:55,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:58,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:42:59,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:00,297 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:00,387 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:01,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:01,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:02,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:43:02,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:43:02,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:43:03,175 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:43:03,528 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:43:03,529 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:43:03,529 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:43:03,530 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:43:04,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:04,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:04,298 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:43:05,298 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:05,530 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:06,299 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:07,788 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:43:07,828 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:07,847 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:10,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:10,705 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:11,503 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:12,301 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:12,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:12,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:14,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:14,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:16,541 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:17,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:43:17,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:43:17,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:43:17,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:17,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:18,303 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:19,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:19,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:21,604 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:21,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:22,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:24,051 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:24,063 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:24,305 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:26,604 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:26,816 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:26,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:28,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:28,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:30,306 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:31,643 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:31,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:31,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:32,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:43:32,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:43:32,735 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:43:33,177 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:43:33,984 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:34,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:36,308 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:36,685 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:37,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:37,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:39,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:39,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:41,755 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:42,310 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:42,499 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:42,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:46,341 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:43:46,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:46,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:46,817 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:47,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:43:47,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:43:47,736 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:43:48,312 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:48,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:48,416 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:51,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:51,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:51,880 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:53,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:53,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:54,314 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:43:56,601 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:56,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:56,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:43:58,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:43:58,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:00,316 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:01,843 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:01,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:01,937 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:02,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:44:02,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:44:02,736 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:44:03,179 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:44:03,886 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:03,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:05,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:05,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:06,318 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:06,979 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:09,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:09,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:11,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:11,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:12,041 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:12,319 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:13,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:13,267 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:16,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:16,293 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:17,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:17,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:44:17,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:44:17,736 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:44:18,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:18,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:18,321 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:21,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:21,327 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:22,182 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:24,323 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:24,890 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:44:24,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:24,920 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:26,940 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:26,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:27,183 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:29,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:29,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:30,325 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:31,962 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:31,973 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:32,242 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:32,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:44:32,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:44:32,736 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:44:33,180 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:44:35,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:35,225 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:36,327 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:37,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:37,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:37,317 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:39,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:39,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:42,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:42,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:42,328 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:42,392 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:44,167 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:44,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:47,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:47,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:47,475 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:47,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:44:47,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:44:47,736 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:44:48,330 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:49,399 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:49,409 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:52,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:52,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:52,762 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:54,332 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:44:54,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:54,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:56,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:56,830 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:57,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:44:58,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:44:58,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:00,334 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:02,590 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:45:02,607 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:02,614 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:02,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:45:02,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:45:02,736 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:45:03,182 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:45:03,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:05,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:05,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:06,336 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:07,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:07,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:07,700 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:09,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:09,718 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:12,338 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:12,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:12,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:12,764 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:14,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:14,503 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:17,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:17,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:17,735 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:45:17,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:45:17,736 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:45:17,909 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:18,340 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:19,402 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:19,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:21,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:21,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:23,122 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:24,342 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:24,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:24,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:26,462 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:26,472 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:28,123 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:28,490 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:28,500 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:31,501 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:31,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:32,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:45:32,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:45:32,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:45:33,184 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:45:33,186 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:33,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:33,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:36,636 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:36,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:38,590 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:40,288 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:45:40,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:40,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:40,346 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 17:45:40,348 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 17:45:40,348 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 17:45:40,349 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 17:45:40,349 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 17:45:40,350 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:41,349 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:42,350 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:43,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:43,343 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:44,351 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:45,366 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:45,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:47,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:45:47,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:45:47,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:45:48,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:48,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:49,964 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:50,230 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:50,244 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:52,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:52,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:54,353 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:45:55,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:55,647 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:55,854 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:45:57,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:45:57,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:00,754 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:00,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:00,854 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:02,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:46:02,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:46:02,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:46:02,785 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:02,796 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:03,186 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:46:05,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:05,759 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:06,357 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:46:06,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:07,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:07,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:10,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:10,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:11,549 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:13,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:13,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:16,232 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:16,243 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:16,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:17,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:46:17,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:46:17,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:46:18,360 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:46:19,988 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:46:20,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:20,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:21,865 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:22,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:22,059 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:24,810 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:24,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:26,860 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:26,866 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:26,873 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:29,720 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:29,730 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:31,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:31,775 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:32,364 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:46:32,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:46:32,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:46:32,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:46:32,938 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:33,192 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:46:34,741 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:34,765 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:36,794 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:36,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:38,194 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:39,977 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:40,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:42,033 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:42,058 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:43,194 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:44,082 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:44,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:44,368 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:46:46,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:46,145 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:47,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:46:47,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:46:47,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:46:48,877 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:49,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:49,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:51,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:51,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:53,878 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:46:54,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:54,080 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:57,654 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:46:57,673 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:57,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:58,373 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:46:59,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:59,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:46:59,858 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:02,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:02,701 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:02,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:47:02,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:47:02,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:47:03,194 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:47:04,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:04,778 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:05,195 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:07,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:07,968 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:10,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:10,022 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:10,893 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:12,072 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:12,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:12,377 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:47:15,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:15,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:15,894 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:17,096 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:17,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:17,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:47:17,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:47:17,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:47:19,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:19,967 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:21,303 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:22,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:22,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:24,381 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:47:24,950 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:24,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:27,004 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:27,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:27,082 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:29,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:29,050 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:31,941 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:31,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:32,083 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:32,736 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:47:32,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:47:32,737 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:47:33,195 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:47:35,547 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:47:35,568 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:35,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:37,609 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:38,385 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:47:38,625 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:38,640 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:40,682 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:40,695 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:42,609 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:43,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:43,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:45,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:45,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:47,610 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:47,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:47:47,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:47:47,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:47:48,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:48,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:50,666 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:50,678 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:52,390 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:47:52,793 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:53,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:53,566 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:55,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:55,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:57,794 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:47:58,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:47:58,401 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:00,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:00,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:02,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:48:02,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:48:02,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:48:02,919 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:03,197 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:48:03,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:03,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:04,393 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:48:05,436 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:05,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:08,198 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:08,382 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:08,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:10,427 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:10,437 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:13,199 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:15,233 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:48:15,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:15,265 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:17,282 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:17,299 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:17,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:48:17,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:48:17,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:48:18,402 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:48:18,970 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:20,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:20,170 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:22,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:22,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:23,970 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:25,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:25,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:27,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:27,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:28,972 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:30,333 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:30,344 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:32,363 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:32,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:32,407 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:48:32,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:48:32,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:48:32,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:48:33,198 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:48:34,200 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:35,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:35,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:37,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:37,212 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:39,200 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:40,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:40,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:42,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:42,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:44,411 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:48:44,517 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:45,026 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:45,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:47,054 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:47,065 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:47,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:48:47,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:48:47,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:48:49,884 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:49,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:49,927 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:53,489 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:48:53,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:53,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:54,928 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:48:56,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:56,555 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:58,416 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:48:58,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:48:58,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:00,656 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:01,616 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:01,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:02,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:49:02,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:49:02,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:49:03,201 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:49:03,723 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:03,736 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:06,202 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:06,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:06,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:08,631 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:08,649 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:10,419 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:49:11,606 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:11,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:11,909 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:13,644 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:13,653 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:16,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:16,667 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:16,910 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:17,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:49:17,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:49:17,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:49:18,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:18,697 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:21,662 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:21,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:22,897 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:23,717 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:23,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:24,424 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:49:26,525 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:26,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:27,898 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:28,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:28,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:31,921 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:49:31,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:31,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:32,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:49:32,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:49:32,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:49:32,923 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:33,202 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:49:34,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:34,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:36,428 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:49:36,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:36,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:38,000 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:39,588 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:39,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:41,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:41,628 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:43,001 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:44,415 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:44,429 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:46,452 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:46,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:47,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:49:47,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:49:47,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:49:48,012 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:49,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:49,407 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:50,433 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:49:51,430 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:51,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:53,020 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:54,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:54,362 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:56,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:56,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:58,020 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:49:59,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:49:59,364 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:01,388 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:01,398 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:02,437 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:50:02,737 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:50:02,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:50:02,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:50:03,203 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:50:03,206 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:04,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:04,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:06,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:06,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:08,205 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:10,326 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:50:10,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:10,366 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:13,206 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:13,597 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:13,622 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:15,645 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:15,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:16,442 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:50:17,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:50:17,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:50:17,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:50:18,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:18,619 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:18,968 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:20,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:20,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:23,440 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:23,463 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:23,968 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:25,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:25,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:28,446 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:50:28,761 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:28,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:29,243 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:30,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:30,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:32,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:50:32,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:50:32,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:50:32,870 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:32,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:33,205 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:50:35,206 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:36,039 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:36,058 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:38,101 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:38,114 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:40,207 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:41,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:41,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:42,450 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:50:43,100 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:43,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:45,602 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:46,173 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:46,189 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:47,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:50:47,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:50:47,738 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:50:49,763 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:50:49,782 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:49,793 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:51,455 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:51,818 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:51,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:53,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:53,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:54,455 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:50:56,684 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:50:56,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:56,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:58,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:50:59,003 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:01,023 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:01,035 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:01,685 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:02,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:51:02,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:51:02,739 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:51:03,207 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:51:04,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:04,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:06,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:06,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:06,832 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:08,460 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:51:08,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:08,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:10,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:10,910 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:11,839 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:13,976 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:13,997 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:16,035 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:16,054 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:16,840 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:17,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:51:17,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:51:17,739 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:51:18,099 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:18,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:20,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:20,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:21,936 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:22,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:22,236 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:22,464 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:51:25,994 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:51:26,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:26,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:26,937 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:28,868 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:28,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:30,898 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:30,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:31,937 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:32,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:51:32,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:51:32,739 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:51:33,209 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:51:34,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:34,048 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:34,468 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:51:36,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:36,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:37,689 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:38,102 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:38,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:41,185 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:41,202 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:42,471 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:51:42,935 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:43,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:43,239 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:45,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:45,277 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:47,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:51:47,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:51:47,739 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:51:48,371 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:48,381 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:48,473 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:51:50,280 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:50,401 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:50,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:53,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:53,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:55,280 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:51:55,408 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:55,420 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:57,444 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:57,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:51:58,476 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:52:00,503 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:00,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:00,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:02,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:52:02,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:52:02,739 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:52:03,210 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:52:04,457 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:52:04,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:04,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:06,211 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:06,531 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:06,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:08,577 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:08,590 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:10,480 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:52:11,538 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:11,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:11,903 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:13,585 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:13,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:16,545 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:16,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:16,904 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:17,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:52:17,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:52:17,739 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:52:18,611 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:18,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:21,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:21,514 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:22,484 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:52:22,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:23,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:23,548 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:26,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:26,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:27,694 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:28,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:28,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:31,776 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:31,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:32,695 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:32,738 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:52:32,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:52:32,739 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:52:33,211 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:52:33,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:33,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:34,488 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:52:36,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:36,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:38,629 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:38,791 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:38,802 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:42,135 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:52:42,154 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:42,166 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:43,629 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:44,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:44,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:46,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:46,925 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:47,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:52:47,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:52:47,739 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:52:48,492 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:52:48,924 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:48,946 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:48,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:51,660 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:51,671 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:53,689 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:53,700 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:53,925 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:52:56,491 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:56,526 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:58,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:58,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:52:58,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:00,495 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:53:01,369 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:01,380 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:02,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:53:02,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:53:02,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:53:03,213 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:53:03,401 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:03,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:04,214 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:06,302 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:06,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:08,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:08,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:09,215 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:11,233 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:11,278 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:13,298 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:13,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:14,500 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:53:14,800 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:16,217 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:16,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:17,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:53:17,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:53:17,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:53:19,781 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:53:19,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:19,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:19,920 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:22,687 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:22,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:24,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:24,748 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:24,921 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:26,509 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:53:27,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:27,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:29,595 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:29,615 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:30,746 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:32,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:32,529 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:32,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:53:32,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:53:32,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:53:33,215 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:53:34,565 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:34,575 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:36,216 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:37,354 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:37,374 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:39,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:39,412 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:40,513 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:53:41,840 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:42,231 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:42,255 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:44,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:44,300 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:46,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:46,331 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:46,841 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:47,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:53:47,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:53:47,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:53:48,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:48,364 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:50,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:50,395 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:51,961 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:52,434 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:52,447 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:52,517 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:53:56,609 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:53:56,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:56,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:57,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:53:58,915 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:53:58,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:01,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:01,724 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:02,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:02,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:54:02,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:54:02,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:54:03,216 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:54:03,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:03,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:06,521 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:54:06,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:06,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:07,171 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:08,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:08,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:10,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:10,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:12,172 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:13,019 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:13,035 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:16,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:16,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:17,173 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:17,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:54:17,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:54:17,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:54:18,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:18,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:18,525 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:54:21,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:21,143 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:23,137 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:23,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:23,174 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:25,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:25,954 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:27,975 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:27,988 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:28,141 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:31,353 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:31,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:32,529 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:54:32,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:54:32,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:54:32,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:54:33,217 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:54:33,220 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:35,090 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:54:35,117 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:35,129 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:37,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:37,160 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:38,219 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:39,182 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:39,197 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:42,133 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:42,144 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:43,220 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:44,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:44,179 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:44,532 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:54:47,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:47,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:47,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:54:47,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:54:47,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:54:48,966 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:49,199 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:49,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:52,124 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:52,136 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:53,967 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:54:54,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:54,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:56,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:56,222 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:58,537 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:54:59,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:59,351 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:54:59,545 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:01,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:01,390 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:02,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:55:02,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:55:02,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:55:03,219 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:55:03,413 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:03,423 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:05,221 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:06,257 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:06,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:08,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:08,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:10,221 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:11,715 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:55:11,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:11,764 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:12,541 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:55:14,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:14,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:15,931 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:16,603 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:16,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:17,739 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:55:17,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:55:17,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:55:18,656 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:18,677 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:21,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:21,809 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:21,898 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:23,841 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:23,851 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:24,545 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:55:25,903 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:25,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:27,127 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:28,856 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:28,875 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:30,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:30,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:32,128 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:32,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:55:32,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:55:32,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:55:33,221 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:55:33,822 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:33,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:35,872 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:35,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:37,223 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:38,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:38,722 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:39,550 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:55:40,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:40,758 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:42,342 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:42,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:42,798 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:44,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:44,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:47,343 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:47,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:55:47,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:55:47,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:55:48,548 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:55:48,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:48,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:51,513 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:51,524 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:52,614 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:53,543 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:53,554 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:55:53,554 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:56,476 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:56,488 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:57,614 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:55:58,508 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:55:58,523 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:01,325 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:01,337 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:02,615 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:02,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:56:02,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:56:02,740 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:56:03,222 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:56:03,359 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:03,373 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:05,558 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:56:05,639 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:05,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:07,674 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:07,680 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:07,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:10,847 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:10,858 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:12,676 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:12,879 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:12,889 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:15,927 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:15,937 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:17,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:56:17,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:56:17,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:56:17,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:17,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:19,563 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:56:19,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:20,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:20,441 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:22,726 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:22,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:25,442 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:26,510 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:56:26,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:26,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:29,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:29,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:30,442 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:31,566 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:56:31,836 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:31,847 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:32,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:56:32,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:56:32,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:56:33,224 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:56:33,883 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:33,895 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:36,226 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:36,978 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:36,998 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:39,020 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:39,038 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:41,226 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:41,846 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:41,866 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:43,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:43,897 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:45,570 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:56:46,539 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:46,709 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:46,729 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:47,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:56:47,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:56:47,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:56:48,752 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:48,769 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:51,905 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:51,951 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:51,970 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:54,018 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:54,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:56,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:56,095 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:57,809 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:56:58,128 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:58,140 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:56:59,574 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:57:00,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:00,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:02,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:57:02,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:57:02,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:57:02,966 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:03,225 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:57:03,510 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:57:03,536 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:03,546 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:06,198 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:06,210 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:08,227 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:08,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:08,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:11,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:11,077 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:11,578 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:57:13,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:13,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:14,108 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:15,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:15,172 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:17,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:57:17,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:57:17,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:57:18,223 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:18,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:19,878 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:20,261 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:20,274 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:22,294 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:22,305 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:24,385 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:24,406 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:25,452 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:25,583 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:57:26,454 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:26,476 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:29,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:29,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:30,452 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:31,519 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:31,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:32,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:57:32,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:57:32,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:57:33,227 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:57:34,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:34,323 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:36,229 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:36,342 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:36,361 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:37,591 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:57:40,154 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:57:41,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:41,178 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:41,559 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:43,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:43,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:45,594 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:57:45,855 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:45,891 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:47,340 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:47,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:57:47,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:57:47,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:57:47,942 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:47,964 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:50,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:50,032 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:52,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:52,829 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:52,853 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:53,597 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:57:54,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:54,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:57,728 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:57:57,884 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:57,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:59,599 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:57:59,938 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:57:59,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:02,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:58:02,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:58:02,741 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:58:02,880 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:02,904 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:03,002 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:03,229 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:58:04,949 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:04,963 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:07,012 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:07,028 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:08,231 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:09,603 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:58:10,184 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:10,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:12,250 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:12,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:13,577 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:14,304 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:14,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:17,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:58:17,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:58:17,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:58:17,955 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:58:17,994 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:18,014 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:20,591 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:21,005 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:21,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:23,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:23,111 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:23,607 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:58:25,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:26,037 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:26,061 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:28,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:28,132 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:30,691 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:31,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:31,273 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:32,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:58:32,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:58:32,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:58:33,230 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:58:33,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:33,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:35,610 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:58:35,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:36,623 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:36,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:38,683 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:38,706 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:40,731 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:40,743 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:40,780 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:42,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:42,819 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:45,732 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:45,957 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:45,991 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:47,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:58:47,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:58:47,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:58:48,015 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:48,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:49,615 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:58:50,091 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:50,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:50,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:53,066 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:53,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:55,927 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:58:57,019 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:58:57,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:57,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:59,115 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:58:59,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:01,889 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:02,045 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:02,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:02,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:59:02,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:59:02,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:59:03,233 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:59:03,619 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:59:04,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:04,134 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:07,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:07,234 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:07,258 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:09,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:09,318 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:11,352 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:11,375 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:12,235 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:14,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:14,579 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:15,622 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:59:16,599 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:16,630 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:17,293 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:17,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:59:17,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:59:17,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:59:18,665 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:18,719 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:21,731 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:21,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:22,975 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:23,800 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:23,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:25,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:25,893 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:28,474 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:28,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:28,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:29,626 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:59:30,909 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:30,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:32,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:59:32,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:59:32,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:59:33,234 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 17:59:34,236 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:34,676 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 17:59:34,704 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:34,715 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:37,659 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:37,684 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:39,236 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:39,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:39,749 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:41,630 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:59:41,787 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:41,813 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:44,613 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:44,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:44,888 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:46,956 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:46,995 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:47,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 17:59:47,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 17:59:47,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 17:59:49,040 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:49,056 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:49,926 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:52,142 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:52,173 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:54,228 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:54,254 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:55,634 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 17:59:55,776 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 17:59:56,288 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:56,317 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:59,284 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 17:59:59,310 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:00,777 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:01,347 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:01,374 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:02,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:00:02,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:00:02,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:00:03,236 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:00:04,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:04,431 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:06,238 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:06,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:06,475 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:07,638 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:00:09,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:09,596 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:11,869 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:13,348 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:00:13,376 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:13,386 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:16,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:16,547 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:16,869 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:17,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:00:17,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:00:17,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:00:18,598 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:18,609 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:20,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:20,654 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:21,642 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:00:21,880 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:23,784 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:23,808 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:25,842 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:25,863 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:26,881 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:27,896 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:27,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:31,073 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:31,108 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:31,881 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:32,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:00:32,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:00:32,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:00:33,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:33,152 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:33,237 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:00:35,175 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:35,203 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:35,646 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:00:37,293 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:38,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:38,289 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:40,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:40,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:42,294 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:42,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:42,441 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:45,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:45,389 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:47,438 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:47,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:47,550 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:47,650 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:00:47,741 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:00:47,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:00:47,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:00:51,215 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:00:51,245 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:51,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:52,940 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:54,279 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:54,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:56,336 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:56,346 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:57,940 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:00:58,383 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:00:58,400 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:01,428 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:01,451 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:01,654 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:01:02,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:01:02,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:01:02,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:01:02,967 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:03,239 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:01:03,477 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:03,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:05,537 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:05,557 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:07,657 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:01:08,325 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:08,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:08,612 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:10,655 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:10,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:13,659 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:01:13,664 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:13,685 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:14,025 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:15,756 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:15,766 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:17,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:01:17,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:01:17,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:01:18,757 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:18,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:19,971 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:20,799 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:20,815 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:22,844 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:24,036 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:25,460 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:25,662 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:01:26,058 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:26,069 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:29,793 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:01:29,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:29,831 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:30,462 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:32,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:01:32,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:01:32,742 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:01:32,908 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:32,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:33,241 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:01:34,981 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:35,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:35,851 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:37,065 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:37,084 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:37,667 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:01:40,055 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:40,083 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:40,858 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:42,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:42,150 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:44,194 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:44,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:45,859 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:47,116 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:47,153 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:47,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:01:47,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:01:47,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:01:49,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:49,242 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:49,672 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:01:51,291 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:52,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:52,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:54,379 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:54,403 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:56,293 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:01:56,442 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:56,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:59,404 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:01:59,430 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:01,459 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:01,473 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:01,614 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:01,675 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:02:02,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:02:02,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:02:02,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:02:03,241 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:02:03,515 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:03,530 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:07,071 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:02:07,110 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:07,122 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:07,243 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:10,044 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:10,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:12,092 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:12,106 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:12,244 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:13,679 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:02:14,131 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:14,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:17,247 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:17,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:17,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:02:17,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:02:17,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:02:17,970 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:19,303 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:19,321 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:22,510 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:22,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:22,970 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:24,582 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:24,592 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:27,646 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:27,679 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:27,683 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:02:28,182 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:29,739 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:29,751 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:31,807 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:31,820 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:32,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:02:32,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:02:32,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:02:33,243 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:02:33,246 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:34,772 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:34,797 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:36,850 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:36,861 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:38,526 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:38,906 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:38,916 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:39,689 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:02:42,021 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:42,049 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:43,526 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:45,863 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:02:45,892 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:45,914 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:47,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:02:47,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:02:47,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:02:47,936 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:47,966 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:48,954 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:49,990 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:50,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:51,694 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:02:52,999 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:53,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:54,938 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:02:55,057 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:55,075 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:58,113 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:58,139 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:02:59,939 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:00,162 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:00,180 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:02,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:03:02,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:03:02,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:03:03,245 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:03:03,348 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:03,369 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:03,698 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:03:05,250 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:05,396 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:05,418 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:07,440 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:07,457 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:10,251 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:10,455 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:10,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:12,504 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:12,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:14,553 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:14,576 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:15,702 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:03:15,740 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:17,520 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:17,542 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:17,742 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:03:17,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:03:17,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:03:19,586 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:19,602 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:20,900 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:23,428 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:03:23,469 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:23,486 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:25,901 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:26,410 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:26,433 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:28,456 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:28,470 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:29,707 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:03:30,493 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:30,517 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:31,179 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:32,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:03:32,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:03:32,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:03:33,247 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:03:33,356 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:33,374 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:35,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:35,443 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:36,248 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:38,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:38,544 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:40,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:40,605 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:41,370 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:41,711 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:03:42,641 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:42,651 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:45,661 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:45,686 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:46,370 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:47,728 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:47,742 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:47,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:03:47,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:03:47,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:03:49,788 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:49,805 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:51,727 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:52,924 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:52,943 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:53,715 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:03:54,982 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:55,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:56,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:03:57,034 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:03:57,062 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:00,615 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:04:00,652 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:00,676 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:01,734 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:02,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:04:02,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:04:02,743 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:04:03,248 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:04:03,836 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:03,859 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:05,720 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:04:05,881 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:05,899 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:06,899 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:08,952 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:08,974 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:11,001 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:11,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:11,900 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:14,046 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:14,068 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:16,089 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:16,103 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:17,179 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:17,724 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:04:17,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:04:17,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:04:17,744 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:04:18,125 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:18,148 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:21,157 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:21,179 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:22,971 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:23,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:23,239 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:25,292 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:25,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:27,972 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:28,291 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:28,320 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:29,728 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:04:30,372 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:30,387 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:32,419 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:32,432 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:32,743 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:04:32,744 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:04:32,744 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:04:33,250 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:04:33,251 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:34,460 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:34,479 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:37,943 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:04:37,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:37,996 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:38,252 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:40,852 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:40,876 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:41,732 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:04:42,913 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:42,929 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:43,816 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:45,961 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:45,983 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:47,744 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:04:47,744 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:04:47,744 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:04:48,016 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:48,030 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:48,972 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:51,132 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:51,151 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:53,176 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:53,196 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:53,736 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:04:54,125 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:04:55,229 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:55,262 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:58,256 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:58,276 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:04:59,126 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:00,297 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:00,313 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:02,335 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:02,350 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:02,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:05:02,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:05:02,745 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:05:03,252 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:05:04,253 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:05,474 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:05,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:07,522 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:07,552 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:07,740 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:05:09,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:09,608 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:10,448 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:12,468 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:12,481 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:14,609 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:16,168 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:05:16,193 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:16,206 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:17,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:05:17,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:05:17,745 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:05:19,240 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:19,270 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:19,745 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:05:19,940 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:21,316 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:21,338 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:24,464 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:24,487 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:24,940 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:26,551 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:26,562 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:29,694 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:29,711 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:29,941 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:31,750 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:05:31,774 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:31,789 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:32,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:05:32,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:05:32,746 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:05:33,254 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:05:34,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:34,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:35,255 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:36,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:37,010 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:40,146 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:40,177 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:40,256 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:42,201 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:42,227 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:43,753 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:05:44,281 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:44,314 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:45,385 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:47,213 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:47,232 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:47,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:05:47,745 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:05:47,745 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:05:49,264 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:49,283 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:50,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:52,290 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:52,309 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:55,757 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:05:55,893 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:05:56,052 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:05:56,081 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:56,104 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:58,126 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:05:58,147 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:00,895 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:01,155 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:01,192 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:02,771 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:06:02,772 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:06:02,787 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:06:03,178 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: partial_history -2024-03-27 18:06:03,179 DEBUG SenderThread:1483352 [sender.py:send():382] send: history -2024-03-27 18:06:03,180 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: summary_record -2024-03-27 18:06:03,182 INFO SenderThread:1483352 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end -2024-03-27 18:06:03,218 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:03,249 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:03,256 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:06:03,760 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/wandb-summary.json -2024-03-27 18:06:03,761 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:06:04,761 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:06:05,269 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:05,298 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:06,257 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:08,495 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:08,527 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:10,550 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:10,812 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:11,259 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:13,764 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:06:13,860 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:14,064 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:16,105 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:16,129 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:16,510 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:17,768 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:06:17,788 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:06:17,788 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:06:19,298 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:19,329 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:21,355 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:21,425 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:22,293 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:24,587 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:24,613 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:26,635 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:26,650 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:27,294 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:28,668 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:29,945 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:31,980 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:32,230 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:32,296 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:32,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:06:32,882 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:06:32,883 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:06:33,550 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:06:35,791 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:06:35,823 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:35,849 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:37,551 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:38,928 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:38,958 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:39,773 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:06:40,985 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:41,006 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:42,968 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:43,029 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:43,047 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:45,087 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:45,116 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:45,775 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:06:47,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:06:47,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:06:47,759 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:06:48,149 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:48,169 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:48,961 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:49,777 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:06:50,234 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:50,246 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:52,312 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:53,556 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:54,380 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:06:55,578 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:55,588 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:55,779 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:06:57,617 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:06:57,632 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:00,319 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:00,507 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:00,534 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:01,781 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:02,564 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:02,583 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:02,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:07:02,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:07:02,759 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:07:03,259 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:07:04,610 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:04,626 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:06,261 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:07,676 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:07,690 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:07,783 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:09,737 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:09,753 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:11,846 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:13,578 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:07:13,600 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:13,620 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:13,785 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:16,734 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:16,755 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:17,428 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:17,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:07:17,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:07:17,759 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:07:18,803 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:18,824 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:19,787 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:20,869 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:20,885 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:23,377 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:23,969 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:24,007 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:25,789 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:26,053 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:26,071 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:28,139 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:28,161 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:28,971 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:31,207 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:31,237 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:31,791 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:32,758 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:07:32,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:07:32,759 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:07:33,261 DEBUG SenderThread:1483352 [sender.py:send():382] send: stats -2024-03-27 18:07:33,334 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:33,367 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:34,262 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:36,482 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:36,516 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:37,793 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:38,572 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:38,585 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:39,684 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:41,781 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:41,792 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:43,795 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:43,836 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:43,857 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:45,540 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:45,878 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:45,890 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:47,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:07:47,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:07:47,759 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status -2024-03-27 18:07:47,919 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:47,930 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:49,797 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:50,959 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:51,510 ERROR gpu :1483352 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero -2024-03-27 18:07:51,560 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:51,573 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:53,798 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:07:54,725 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:54,760 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:56,033 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:07:56,824 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:56,840 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:58,886 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:58,922 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:07:59,800 INFO Thread-13 :1483352 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_141031-aoxf8fxn/files/output.log -2024-03-27 18:08:01,917 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:01,944 ERROR gpu :1483352 [interfaces.py:monitor():144] Failed to sample metric: Not Supported -2024-03-27 18:08:01,988 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: status_report -2024-03-27 18:08:02,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: stop_status -2024-03-27 18:08:02,759 DEBUG HandlerThread:1483352 [handler.py:handle_request():146] handle_request: internal_messages -2024-03-27 18:08:02,760 DEBUG SenderThread:1483352 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:04:18,566 INFO StreamThr :1893870 [internal.py:wandb_internal():86] W&B internal server running at pid: 1893870, started at: 2024-03-27 19:04:18.565290 +2024-03-27 19:04:18,567 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status +2024-03-27 19:04:18,570 INFO WriterThread:1893870 [datastore.py:open_for_write():87] open: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/run-lwtbcr8s.wandb +2024-03-27 19:04:18,570 DEBUG SenderThread:1893870 [sender.py:send():382] send: header +2024-03-27 19:04:18,640 DEBUG SenderThread:1893870 [sender.py:send():382] send: run +2024-03-27 19:04:18,989 INFO SenderThread:1893870 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files +2024-03-27 19:04:18,989 INFO SenderThread:1893870 [sender.py:_start_run_threads():1136] run started: lwtbcr8s with start time 1711562658.56459 +2024-03-27 19:04:18,996 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: check_version +2024-03-27 19:04:18,996 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: check_version +2024-03-27 19:04:19,037 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: run_start +2024-03-27 19:04:19,050 DEBUG HandlerThread:1893870 [system_info.py:__init__():27] System info init +2024-03-27 19:04:19,050 DEBUG HandlerThread:1893870 [system_info.py:__init__():42] System info init done +2024-03-27 19:04:19,050 INFO HandlerThread:1893870 [system_monitor.py:start():194] Starting system monitor +2024-03-27 19:04:19,050 INFO SystemMonitor:1893870 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-27 19:04:19,051 INFO HandlerThread:1893870 [system_monitor.py:probe():214] Collecting system info +2024-03-27 19:04:19,051 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started cpu monitoring +2024-03-27 19:04:19,052 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started disk monitoring +2024-03-27 19:04:19,053 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started gpu monitoring +2024-03-27 19:04:19,054 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started memory monitoring +2024-03-27 19:04:19,056 INFO SystemMonitor:1893870 [interfaces.py:start():190] Started network monitoring +2024-03-27 19:04:19,090 DEBUG HandlerThread:1893870 [system_info.py:probe():151] Probing system +2024-03-27 19:04:19,096 DEBUG HandlerThread:1893870 [system_info.py:_probe_git():136] Probing git +2024-03-27 19:04:19,107 DEBUG HandlerThread:1893870 [system_info.py:_probe_git():144] Probing git done +2024-03-27 19:04:19,107 DEBUG HandlerThread:1893870 [system_info.py:probe():199] Probing system done +2024-03-27 19:04:19,108 DEBUG HandlerThread:1893870 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-03-27T18:04:19.090782', 'startedAt': '2024-03-27T18:04:18.560848', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=distil-whisper/distil-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_1', '--dataset_config_name=hi', '--language=hindi', '--train_split_name=train+validation', '--eval_split_name=test', '--max_steps=5000', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=32', '--logging_steps=25', '--learning_rate=1e-4', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--save_total_limit=1', '--generation_max_length=225', '--preprocessing_num_workers=1', '--dataloader_num_workers=4', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--gradient_checkpointing', '--fp16', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-large-v3-hi-ft', 'commit': '40c686df113c0e98e7363c1bd523f58d11848fc0'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/distil-large-v3-hi-ft', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2290.4996484375006, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2186.729, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.542, 'min': 1500.0, 'max': 2250.0}, {'current': 1792.045, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.339, 'min': 1500.0, 'max': 2250.0}, {'current': 1791.496, 'min': 1500.0, 'max': 2250.0}, {'current': 1742.02, 'min': 1500.0, 'max': 2250.0}, {'current': 1692.095, 'min': 1500.0, 'max': 2250.0}, {'current': 1658.911, 'min': 1500.0, 'max': 2250.0}, {'current': 1606.212, 'min': 1500.0, 'max': 2250.0}, {'current': 1606.808, 'min': 1500.0, 'max': 2250.0}, {'current': 1606.448, 'min': 1500.0, 'max': 2250.0}, {'current': 2094.446, 'min': 1500.0, 'max': 2250.0}, {'current': 1734.944, 'min': 1500.0, 'max': 2250.0}, {'current': 1736.215, 'min': 1500.0, 'max': 2250.0}, {'current': 2489.415, 'min': 1500.0, 'max': 2250.0}, {'current': 1737.217, 'min': 1500.0, 'max': 2250.0}, {'current': 1695.232, 'min': 1500.0, 'max': 2250.0}, {'current': 2416.542, 'min': 1500.0, 'max': 2250.0}, {'current': 1737.614, 'min': 1500.0, 'max': 2250.0}, {'current': 1599.452, 'min': 1500.0, 'max': 2250.0}, {'current': 1772.758, 'min': 1500.0, 'max': 2250.0}, {'current': 1738.087, 'min': 1500.0, 'max': 2250.0}, {'current': 2311.378, 'min': 1500.0, 'max': 2250.0}, {'current': 1735.599, 'min': 1500.0, 'max': 2250.0}, {'current': 1620.967, 'min': 1500.0, 'max': 2250.0}, {'current': 1864.863, 'min': 1500.0, 'max': 2250.0}, {'current': 1729.177, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.804, 'min': 1500.0, 'max': 2250.0}, {'current': 1788.133, 'min': 1500.0, 'max': 2250.0}, {'current': 1795.255, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.306, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.529, 'min': 1500.0, 'max': 2250.0}, {'current': 1733.742, 'min': 1500.0, 'max': 2250.0}, {'current': 2473.813, 'min': 1500.0, 'max': 2250.0}, {'current': 1932.309, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.805, 'min': 1500.0, 'max': 2250.0}, {'current': 1730.935, 'min': 1500.0, 'max': 2250.0}, {'current': 1729.65, 'min': 1500.0, 'max': 2250.0}, {'current': 1730.989, 'min': 1500.0, 'max': 2250.0}, {'current': 2002.554, 'min': 1500.0, 'max': 2250.0}, {'current': 2089.927, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.843, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.434, 'min': 1500.0, 'max': 2250.0}, {'current': 2271.01, 'min': 1500.0, 'max': 2250.0}, {'current': 3370.134, 'min': 1500.0, 'max': 2250.0}, {'current': 1687.104, 'min': 1500.0, 'max': 2250.0}, {'current': 1688.345, 'min': 1500.0, 'max': 2250.0}, {'current': 1694.029, 'min': 1500.0, 'max': 2250.0}, {'current': 3295.66, 'min': 1500.0, 'max': 2250.0}, {'current': 1684.621, 'min': 1500.0, 'max': 2250.0}, {'current': 1686.668, 'min': 1500.0, 'max': 2250.0}, {'current': 1687.585, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.907, 'min': 1500.0, 'max': 2250.0}, {'current': 1728.023, 'min': 1500.0, 'max': 2250.0}, {'current': 1727.835, 'min': 1500.0, 'max': 2250.0}, {'current': 2090.043, 'min': 1500.0, 'max': 2250.0}, {'current': 2245.903, 'min': 1500.0, 'max': 2250.0}, {'current': 2202.881, 'min': 1500.0, 'max': 2250.0}, {'current': 2284.838, 'min': 1500.0, 'max': 2250.0}, {'current': 1598.829, 'min': 1500.0, 'max': 2250.0}, {'current': 1656.082, 'min': 1500.0, 'max': 2250.0}, {'current': 1661.638, 'min': 1500.0, 'max': 2250.0}, {'current': 1649.714, 'min': 1500.0, 'max': 2250.0}, {'current': 1655.626, 'min': 1500.0, 'max': 2250.0}, {'current': 3027.152, 'min': 1500.0, 'max': 2250.0}, {'current': 2017.291, 'min': 1500.0, 'max': 2250.0}, {'current': 2012.466, 'min': 1500.0, 'max': 2250.0}, {'current': 1990.236, 'min': 1500.0, 'max': 2250.0}, {'current': 2114.394, 'min': 1500.0, 'max': 2250.0}, {'current': 2095.433, 'min': 1500.0, 'max': 2250.0}, {'current': 2096.911, 'min': 1500.0, 'max': 2250.0}, {'current': 1840.403, 'min': 1500.0, 'max': 2250.0}, {'current': 2203.59, 'min': 1500.0, 'max': 2250.0}, {'current': 2185.592, 'min': 1500.0, 'max': 2250.0}, {'current': 2201.94, 'min': 1500.0, 'max': 2250.0}, {'current': 2018.775, 'min': 1500.0, 'max': 2250.0}, {'current': 2200.874, 'min': 1500.0, 'max': 2250.0}, {'current': 1873.351, 'min': 1500.0, 'max': 2250.0}, {'current': 3369.243, 'min': 1500.0, 'max': 2250.0}, {'current': 2235.133, 'min': 1500.0, 'max': 2250.0}, {'current': 3367.352, 'min': 1500.0, 'max': 2250.0}, {'current': 3364.254, 'min': 1500.0, 'max': 2250.0}, {'current': 2202.453, 'min': 1500.0, 'max': 2250.0}, {'current': 2038.768, 'min': 1500.0, 'max': 2250.0}, {'current': 2233.914, 'min': 1500.0, 'max': 2250.0}, {'current': 2182.285, 'min': 1500.0, 'max': 2250.0}, {'current': 2302.527, 'min': 1500.0, 'max': 2250.0}, {'current': 2185.247, 'min': 1500.0, 'max': 2250.0}, {'current': 2132.843, 'min': 1500.0, 'max': 2250.0}, {'current': 2971.398, 'min': 1500.0, 'max': 2250.0}, {'current': 2055.752, 'min': 1500.0, 'max': 2250.0}, {'current': 1567.353, 'min': 1500.0, 'max': 2250.0}, {'current': 1609.825, 'min': 1500.0, 'max': 2250.0}, {'current': 1606.242, 'min': 1500.0, 'max': 2250.0}, {'current': 1610.589, 'min': 1500.0, 'max': 2250.0}, {'current': 1617.499, 'min': 1500.0, 'max': 2250.0}, {'current': 2398.668, 'min': 1500.0, 'max': 2250.0}, {'current': 3020.272, 'min': 1500.0, 'max': 2250.0}, {'current': 2053.074, 'min': 1500.0, 'max': 2250.0}, {'current': 2141.083, 'min': 1500.0, 'max': 2250.0}, {'current': 2182.762, 'min': 1500.0, 'max': 2250.0}, {'current': 2190.047, 'min': 1500.0, 'max': 2250.0}, {'current': 2209.589, 'min': 1500.0, 'max': 2250.0}, {'current': 2072.985, 'min': 1500.0, 'max': 2250.0}, {'current': 2427.726, 'min': 1500.0, 'max': 2250.0}, {'current': 2205.137, 'min': 1500.0, 'max': 2250.0}, {'current': 2197.942, 'min': 1500.0, 'max': 2250.0}, {'current': 2208.468, 'min': 1500.0, 'max': 2250.0}, {'current': 3307.897, 'min': 1500.0, 'max': 2250.0}, {'current': 2210.539, 'min': 1500.0, 'max': 2250.0}, {'current': 2580.208, 'min': 1500.0, 'max': 2250.0}, {'current': 2487.416, 'min': 1500.0, 'max': 2250.0}, {'current': 3360.967, 'min': 1500.0, 'max': 2250.0}, {'current': 2409.573, 'min': 1500.0, 'max': 2250.0}, {'current': 2399.184, 'min': 1500.0, 'max': 2250.0}, {'current': 2400.551, 'min': 1500.0, 'max': 2250.0}, {'current': 2453.642, 'min': 1500.0, 'max': 2250.0}, {'current': 1814.426, 'min': 1500.0, 'max': 2250.0}, {'current': 2440.557, 'min': 1500.0, 'max': 2250.0}, {'current': 2422.632, 'min': 1500.0, 'max': 2250.0}, {'current': 1919.862, 'min': 1500.0, 'max': 2250.0}, {'current': 2519.195, 'min': 1500.0, 'max': 2250.0}, {'current': 1666.938, 'min': 1500.0, 'max': 2250.0}, {'current': 1669.157, 'min': 1500.0, 'max': 2250.0}, {'current': 1794.328, 'min': 1500.0, 'max': 2250.0}, {'current': 1797.233, 'min': 1500.0, 'max': 2250.0}, {'current': 1793.218, 'min': 1500.0, 'max': 2250.0}, {'current': 1790.531, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1614.0720481872559}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5396919250488}} +2024-03-27 19:04:19,108 INFO HandlerThread:1893870 [system_monitor.py:probe():224] Finished collecting system info +2024-03-27 19:04:19,108 INFO HandlerThread:1893870 [system_monitor.py:probe():227] Publishing system info +2024-03-27 19:04:19,109 INFO HandlerThread:1893870 [system_monitor.py:probe():229] Finished publishing system info +2024-03-27 19:04:19,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:19,113 DEBUG SenderThread:1893870 [sender.py:send():382] send: files +2024-03-27 19:04:19,113 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-27 19:04:19,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: python_packages +2024-03-27 19:04:19,118 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: python_packages +2024-03-27 19:04:19,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:04:19,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:04:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:04:19,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:19,280 DEBUG SenderThread:1893870 [sender.py:send():382] send: telemetry +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: config +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: telemetry +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:04:19,281 WARNING SenderThread:1893870 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-03-27 19:04:19,281 DEBUG SenderThread:1893870 [sender.py:send():382] send: telemetry +2024-03-27 19:04:19,625 INFO wandb-upload_0:1893870 [upload_job.py:push():131] Uploaded file /tmp/tmp4zjika_rwandb/yd0vpcfo-wandb-metadata.json +2024-03-27 19:04:19,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-metadata.json +2024-03-27 19:04:19,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/requirements.txt +2024-03-27 19:04:19,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:21,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:21,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:21,990 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:23,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:23,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:24,283 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:26,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:26,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:28,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:28,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:29,285 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:31,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:31,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:33,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:33,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:33,993 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:34,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:04:34,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:04:34,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:04:34,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:35,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:35,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:37,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:37,751 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:37,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:39,995 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:40,267 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:40,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:40,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:42,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:42,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:43,996 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:45,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:45,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:45,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:45,997 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:47,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:47,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:49,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:04:49,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:04:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:04:49,998 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:50,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:50,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:50,804 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:50,999 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/config.yaml +2024-03-27 19:04:51,999 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:52,541 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:52,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:56,001 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:56,107 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:04:56,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:56,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:56,561 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:04:58,002 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:04:58,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:04:58,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:00,002 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:00,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:00,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:02,026 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:02,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:02,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:04,003 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:04,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:04,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:04,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:05,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:05,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:06,004 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:07,567 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:07,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:07,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:09,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:09,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:10,005 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:11,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:11,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:12,005 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:13,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:14,006 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:14,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:14,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:16,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:16,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:18,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:18,608 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:19,057 DEBUG SystemMonitor:1893870 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-27 19:05:19,059 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:05:19,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:19,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:19,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:19,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:20,008 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:21,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:21,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:23,724 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:24,009 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:24,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:24,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:26,009 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:26,712 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:26,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:28,010 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:29,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:29,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:29,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:33,689 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:05:33,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:33,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:34,011 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:34,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:34,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:34,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:34,300 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:36,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:36,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:38,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:38,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:40,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:40,219 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:41,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:41,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:43,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:43,751 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:44,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:46,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:46,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:46,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:48,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:48,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:49,061 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:05:49,118 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:05:49,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:05:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:05:50,015 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:50,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:50,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:51,270 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:53,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:53,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:55,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:55,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:56,016 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:05:57,202 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:05:58,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:05:58,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:00,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:00,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:01,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:06:01,277 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:06:01,278 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:06:01,279 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:06:02,017 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:02,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:06:02,281 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:03,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:03,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:04,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:04,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:04,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:04,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:05,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:05,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:08,019 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:08,224 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:08,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:08,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:12,549 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:06:12,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:12,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:13,225 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:14,021 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:15,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:15,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:17,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:17,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:18,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:19,063 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:06:19,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:19,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:20,022 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:20,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:20,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:22,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:22,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:23,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:25,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:25,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:26,023 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:27,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:27,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:28,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:29,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/config.yaml +2024-03-27 19:06:30,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:30,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:32,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:32,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:32,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:34,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:34,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:34,262 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:35,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:35,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:37,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:37,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:38,026 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:39,427 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:40,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:40,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:42,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:42,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:44,028 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:44,460 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:45,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:45,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:47,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:47,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:49,064 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:06:49,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:06:49,119 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:06:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:06:49,518 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:51,181 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:06:51,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:51,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:52,029 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:54,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:54,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:54,519 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:06:56,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:56,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:58,031 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:06:59,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:59,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:06:59,707 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:01,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:01,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:04,032 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:04,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:04,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:04,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:04,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:04,223 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:05,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:06,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:06,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:09,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:09,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:10,034 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:10,797 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:11,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:11,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:14,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:14,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:15,851 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:16,035 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:16,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:16,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:19,066 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:07:19,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:19,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:19,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:19,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:20,900 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:21,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:21,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:22,037 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:24,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:24,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:25,901 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:26,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:26,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:28,038 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:30,090 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:07:30,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:30,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:30,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:32,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:32,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:34,040 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:34,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:34,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:34,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:34,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:36,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:37,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:37,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:39,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:39,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:40,042 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:41,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:42,051 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:42,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:44,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:44,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:46,043 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:47,105 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:48,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:48,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:49,068 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:07:49,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:07:49,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:07:49,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:07:50,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:50,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:52,045 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:52,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:53,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:53,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:55,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:55,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:57,227 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:07:58,047 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:07:58,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:07:58,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:00,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:00,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:02,227 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:03,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:03,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:04,049 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:04,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:04,121 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:05,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:05,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:07,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:08,731 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:08:08,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:08,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:10,051 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:11,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:11,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:13,318 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:13,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:13,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:16,052 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:16,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:16,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:18,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:18,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:18,613 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:19,069 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:08:19,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:19,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:19,121 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:21,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:21,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:22,054 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:23,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:23,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:23,687 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:26,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:26,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:28,056 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:28,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:28,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:28,739 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:31,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:31,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:32,869 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:08:32,870 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:08:32,870 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:08:32,871 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:08:33,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:08:33,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:33,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:33,871 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:34,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:34,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:34,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:36,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:36,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:38,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:38,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:39,862 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:40,060 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:41,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:41,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:43,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:43,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:44,863 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:46,062 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:46,988 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:08:47,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:47,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:49,071 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:08:49,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:08:49,121 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:08:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:08:50,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:50,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:50,287 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:52,064 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:52,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:52,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:54,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:54,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:55,968 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:08:56,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:56,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:58,066 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:08:58,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:08:58,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:01,030 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:01,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:01,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:03,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:03,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:04,068 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:04,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:06,347 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:06,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:06,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:08,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:08,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:10,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:11,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:11,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:12,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:13,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:13,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:15,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:15,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:16,071 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:17,205 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:18,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:18,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:19,073 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:09:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:19,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:20,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:20,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:22,075 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:22,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:24,380 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:09:24,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:24,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:27,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:27,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:27,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:28,077 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:29,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:29,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:32,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:32,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:32,477 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:34,079 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:34,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:34,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:34,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:37,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:37,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:38,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:39,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:39,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:40,081 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:42,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:42,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:43,588 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:44,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:44,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:46,083 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:47,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:47,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:48,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:49,075 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:09:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:09:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:09:49,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:09:49,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:49,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:52,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:52,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:53,710 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:54,085 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:09:54,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:54,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:56,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:56,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:58,771 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:09:59,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:09:59,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:00,087 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:03,234 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:10:03,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:03,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:03,772 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:04,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:06,089 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:06,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:06,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:08,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:08,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:09,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:10,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:10,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:12,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:13,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:13,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:14,883 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:15,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:15,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,094 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:18,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:18,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:19,076 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:10:19,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:19,121 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:19,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:20,241 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:20,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:20,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:23,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:23,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:24,096 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:25,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:25,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:25,375 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:27,309 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:27,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,098 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:30,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:30,433 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:32,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:32,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:34,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:34,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:34,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:34,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:35,479 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:36,100 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:37,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:37,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:40,480 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:40,798 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:10:40,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:40,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:42,102 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:42,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:42,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:45,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:45,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:45,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:47,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:47,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:48,104 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:49,078 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:10:49,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:10:49,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:10:49,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:10:49,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:49,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:51,234 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:52,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:52,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:54,107 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:10:54,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:54,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:56,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:56,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:56,775 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:10:59,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:10:59,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:00,109 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:01,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:01,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:01,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:04,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:04,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:04,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:04,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:04,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:05,256 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:11:05,258 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:11:05,258 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:11:05,258 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:11:06,111 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:11:06,111 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:06,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:06,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:07,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:08,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:08,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:11,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:11,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:12,113 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:12,502 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:13,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:13,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:16,902 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:11:16,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:16,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:17,503 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:18,115 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:19,080 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:11:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:19,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:19,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:19,771 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:21,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:21,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:23,319 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:24,117 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:24,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:24,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:26,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:26,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:28,626 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:29,865 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:29,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:30,119 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:31,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:31,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:33,986 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:34,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:34,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:34,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:36,121 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:36,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:36,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:39,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:39,038 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:39,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:40,122 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:41,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:41,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:44,109 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:44,124 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:46,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:46,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:46,124 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:48,125 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:48,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:48,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:49,080 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:11:49,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:11:49,122 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:11:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:11:50,800 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:51,028 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:51,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:52,126 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:54,631 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:11:54,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:54,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:55,880 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:11:57,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:57,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:58,129 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:11:59,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:11:59,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:01,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:01,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:01,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:04,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:04,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:04,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:04,131 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:04,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:04,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:06,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:06,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:07,276 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:08,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:08,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:10,133 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:11,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:11,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:12,629 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:13,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:13,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:16,135 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:16,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:16,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:17,689 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:18,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:18,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:19,084 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:12:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:19,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:21,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:21,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:22,137 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:22,969 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:23,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:23,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:26,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:26,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:28,019 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:28,139 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:28,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:28,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:33,004 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:12:33,020 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:33,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:33,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:34,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:34,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:34,141 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:35,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:35,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:37,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:37,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:38,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:40,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:40,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:40,144 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:42,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:42,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:44,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:44,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:44,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:46,146 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:46,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:46,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:49,084 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:12:49,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:12:49,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:12:49,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:12:49,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:49,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:50,941 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:51,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:51,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:52,148 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:55,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:55,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:56,775 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:12:57,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:57,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:58,150 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:12:59,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:12:59,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:01,834 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:02,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:02,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:04,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:04,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:04,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:04,152 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:05,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:05,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:07,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:07,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:07,293 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:10,682 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:13:10,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:10,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:12,155 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:12,961 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:13,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:13,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:15,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:15,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:17,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:17,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:18,009 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:18,157 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:19,085 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:13:19,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:19,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:20,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:20,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:22,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:13:22,132 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:13:22,133 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:13:22,135 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:13:22,159 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:13:22,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:22,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:23,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:24,159 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:25,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:25,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:27,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:27,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:29,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:30,161 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:30,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:30,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:32,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:32,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:34,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:34,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:35,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:35,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:36,163 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:37,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:37,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:39,347 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:40,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:40,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:42,166 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:42,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:42,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:44,465 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:45,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:45,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:48,168 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:49,087 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:13:49,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:13:49,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:13:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:13:49,411 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:13:49,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:49,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:50,264 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:52,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:52,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:54,170 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:13:54,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:54,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:55,582 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:13:57,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:57,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:59,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:13:59,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:00,172 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:00,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:02,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:02,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:04,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:04,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:04,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:04,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:04,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:05,698 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:06,174 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:07,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:07,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:09,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:09,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:10,699 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:12,175 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:12,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:12,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:14,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:14,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:15,772 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:17,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:17,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:18,178 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:19,089 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:14:19,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:19,123 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:19,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:19,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:21,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:22,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:22,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:24,180 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:24,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:24,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:26,888 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:28,076 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:14:28,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:28,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:30,182 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:30,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:30,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:31,936 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:32,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:32,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:34,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:34,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:35,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:35,837 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:36,184 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:36,988 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:37,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:37,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:40,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:40,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:42,056 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:42,186 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:42,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:42,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:45,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:45,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:47,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:47,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:47,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:48,188 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:49,091 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:14:49,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:14:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:14:49,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:14:50,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:51,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:52,529 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:14:53,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:53,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:54,190 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:14:56,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:56,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:14:58,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:00,192 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:00,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:00,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:03,244 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:04,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:04,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:04,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:06,423 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:15:06,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:06,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:07,195 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:08,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:09,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:09,152 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:11,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:11,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:13,197 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:13,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:14,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:14,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:16,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:16,217 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:18,428 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:19,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:19,092 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:15:19,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:19,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:19,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:19,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:21,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:21,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:23,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:24,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:24,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:25,201 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:26,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:26,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:28,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:28,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:28,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:31,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:31,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:31,203 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:33,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:33,551 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:34,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:34,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:34,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:36,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:36,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:37,206 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:38,613 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:39,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:39,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:41,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:41,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:43,209 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:43,676 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:44,609 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:15:44,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:44,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:47,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:47,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:48,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:49,093 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:15:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:15:49,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:15:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:15:49,212 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:49,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:49,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:52,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:52,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:53,878 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:53,950 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:15:53,951 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:15:53,952 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:15:53,952 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:15:54,214 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:15:54,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:54,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:55,214 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:56,220 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:15:57,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:57,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:58,953 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:15:59,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:15:59,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:01,222 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:02,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:02,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:04,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:04,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:04,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:04,246 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:04,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:04,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:06,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:06,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:07,225 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:09,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:09,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:10,013 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:11,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:11,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:13,228 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:14,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:14,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:15,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:16,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:16,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:19,095 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:16:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:19,124 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:19,230 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:19,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:19,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:20,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:23,037 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:16:23,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:23,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:26,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:27,233 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:28,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:28,203 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:31,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:31,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:31,245 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:33,235 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:33,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:33,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:34,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:34,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:36,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:36,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:36,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:38,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:38,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:39,243 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:41,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:41,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:41,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:43,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:43,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:45,245 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:46,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:46,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:46,367 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:48,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:48,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:49,096 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:16:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:16:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:16:49,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:16:51,247 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:51,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:51,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:52,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:53,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:53,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:56,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:56,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:57,250 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:16:57,478 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:16:58,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:16:58,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:01,917 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:17:01,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:01,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:02,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:03,252 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:04,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:04,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:04,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:04,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:06,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:06,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:07,621 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:09,255 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:09,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:09,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:11,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:11,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:12,622 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:14,961 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:14,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:15,257 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:16,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:17,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:17,678 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:19,098 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:17:19,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:19,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:19,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:19,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:21,259 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:21,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:21,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:23,356 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:24,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:24,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:26,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:27,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:27,261 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:28,831 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:29,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:29,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:31,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:32,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:33,265 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:33,893 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:34,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:34,125 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:34,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:34,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:37,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:37,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:38,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:39,267 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:40,552 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:17:40,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:40,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:43,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:43,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:44,009 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:45,269 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:45,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:45,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:48,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:48,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:49,100 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:17:49,101 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:49,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:17:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:17:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:17:49,271 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:50,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:50,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:53,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:53,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:54,331 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:17:55,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:55,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:57,273 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:17:58,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:58,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:17:59,331 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:00,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:00,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:01,274 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:03,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:03,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:04,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:04,341 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:05,276 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:05,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:05,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:07,276 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:08,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:08,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:10,341 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:10,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:10,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:11,099 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:18:11,100 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:18:11,101 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:18:11,102 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:18:11,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:18:11,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:12,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:13,279 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:13,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:13,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:15,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:15,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:16,103 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:19,101 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:18:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:19,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:19,248 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:18:19,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:19,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:19,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:21,280 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:22,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:22,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:23,287 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:24,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:24,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:27,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:27,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:27,247 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:29,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:29,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:29,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:32,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:32,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:32,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:34,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:34,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:34,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:36,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:36,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:37,291 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:37,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:38,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:38,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:41,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:41,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:42,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:43,293 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:44,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:44,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:47,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:47,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:47,508 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:49,102 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:18:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:18:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:18:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:18:49,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:49,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:49,295 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:52,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:52,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:52,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:54,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:54,351 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:55,297 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:18:57,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:18:57,754 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:18:57,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:18:57,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:00,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:00,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:01,299 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:02,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:02,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:02,612 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:04,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:05,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:05,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:07,301 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:07,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:07,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:07,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:10,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:10,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:12,731 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:13,303 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:15,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:15,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:17,800 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:17,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:17,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:19,104 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:19:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:19,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:19,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:19,305 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:20,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:20,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:22,854 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:22,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:22,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:24,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:24,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:25,306 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:27,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:27,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:27,855 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:29,865 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:29,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:31,308 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:31,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:31,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:32,913 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:34,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:34,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:35,284 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:19:35,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:35,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:37,310 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:37,990 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:38,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:38,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:40,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:40,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:42,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:42,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:43,034 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:43,312 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:44,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:44,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:47,001 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:47,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:48,087 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:49,106 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:19:49,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:19:49,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:19:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:19:49,314 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:50,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:50,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:52,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:52,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:53,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:19:55,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:55,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:55,315 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:19:57,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:57,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:19:58,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:00,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:00,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:01,317 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:02,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:02,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:03,220 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:04,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:04,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:04,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:04,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:06,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:06,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:07,323 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:08,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:08,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:08,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:12,526 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:20:12,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:12,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:13,325 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:13,340 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:15,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:15,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:17,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:17,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:18,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:19,107 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:20:19,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:19,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:19,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:19,327 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:20,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:20,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:22,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:22,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:23,464 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:25,330 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:25,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:25,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:27,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:27,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:28,464 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:30,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:30,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:31,332 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:32,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:32,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:33,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:34,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:35,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:35,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:37,334 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:37,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:37,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:38,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:40,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:40,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:41,719 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:20:41,720 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:20:41,721 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:20:41,721 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:20:42,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:20:42,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:42,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:42,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:43,336 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:43,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:45,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:45,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:47,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:47,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:49,108 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:20:49,109 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:49,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:20:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:20:49,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:20:49,339 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:51,501 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:20:51,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:51,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:54,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:54,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:54,766 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:20:55,341 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:20:56,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:56,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:58,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:58,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:20:59,767 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:01,342 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:01,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:01,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:03,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:03,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:04,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:05,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:06,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:06,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:07,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:08,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:08,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:10,888 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:11,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:11,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:13,347 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:13,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:13,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:15,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:16,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:16,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:18,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:18,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:19,110 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:21:19,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:19,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:19,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:19,349 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:21,334 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:21,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:21,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:23,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:23,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:25,351 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:26,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:26,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:27,060 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:30,537 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:21:30,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:30,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:31,353 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:32,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:33,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:33,595 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:34,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:35,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:35,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:37,199 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:37,355 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:38,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:38,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:40,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:40,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:42,200 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:43,357 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:43,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:43,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:45,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:45,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:47,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:48,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:48,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:49,111 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:21:49,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:21:49,126 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:21:49,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:21:49,359 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:50,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:50,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:52,326 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:53,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:53,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:55,361 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:21:55,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:55,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:57,385 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:21:57,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:57,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:21:59,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:00,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:01,363 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:02,452 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:03,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:03,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:04,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:04,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:05,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:05,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:07,366 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:07,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:08,705 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:22:08,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:08,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:11,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:11,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:12,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:13,368 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:13,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:13,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:16,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:16,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:17,597 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:18,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:18,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:19,113 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:22:19,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:19,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:20,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:20,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:21,371 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:23,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:24,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:24,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:26,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:26,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:27,373 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:28,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:28,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:28,740 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:31,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:31,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:33,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:33,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:33,375 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:33,811 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:34,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:34,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:36,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:36,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:38,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:38,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:38,865 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:39,377 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:41,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:41,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:43,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:43,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:43,955 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:45,379 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:47,324 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:22:47,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:47,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:48,955 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:49,115 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:22:49,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:22:49,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:22:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:22:49,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:49,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:51,381 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:51,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:51,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:54,254 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:22:54,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:54,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:56,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:56,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:57,383 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:22:59,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:22:59,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:00,098 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:01,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:01,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:03,385 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:04,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:04,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:04,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:04,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:05,256 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:06,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:06,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:08,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:08,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:09,387 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:10,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:10,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:11,253 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:12,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:12,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:13,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:23:13,408 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:23:13,409 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:23:13,410 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:23:14,389 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:23:15,390 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:15,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:15,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:16,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:17,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:17,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:19,116 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:23:19,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:19,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:20,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:20,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:21,392 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:21,432 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:24,408 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:23:24,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:24,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:26,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:27,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:27,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:27,394 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:29,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:29,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:31,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:31,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:31,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:33,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:33,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:33,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:34,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:34,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:36,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:36,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:37,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:38,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:38,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:39,399 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:41,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:41,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:42,690 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:43,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:43,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:45,401 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:46,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:46,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:47,784 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:48,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:48,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:49,118 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:23:49,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:23:49,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:23:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:23:51,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:51,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:51,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:53,674 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:53,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:53,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:57,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:57,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:57,405 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:23:58,953 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:23:59,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:23:59,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:02,540 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:24:02,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:02,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:03,408 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:04,032 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:04,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:04,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:05,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:05,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:07,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:07,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:09,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:09,410 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:10,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:10,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:12,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:12,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:14,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:15,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:15,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:15,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:17,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:17,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:19,119 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:24:19,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:19,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:19,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:20,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:20,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:22,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:22,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:23,416 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:24,797 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:25,417 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:25,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:25,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:27,417 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:27,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:27,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:29,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:29,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:30,300 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:31,419 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:32,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:32,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:33,420 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:34,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:34,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:34,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:34,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:34,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:36,266 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:36,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:36,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:37,421 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:39,941 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:24:39,968 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:39,983 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:41,764 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:42,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:42,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:43,424 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:44,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:44,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:47,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:47,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:47,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:49,121 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:24:49,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:24:49,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:24:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:24:49,426 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:49,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:49,879 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:52,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:52,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:52,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:54,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:54,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:55,429 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:24:57,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:24:57,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:57,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:59,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:24:59,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:01,430 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:02,671 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:03,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:03,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:04,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:04,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:04,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:05,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:05,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:07,433 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:07,743 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:08,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:08,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:10,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:10,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:13,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:13,435 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:15,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:15,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:18,067 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:18,626 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:25:18,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:18,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:19,122 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:25:19,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:19,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:19,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:21,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:21,073 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:23,090 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:23,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:23,330 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:25,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:25,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:27,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:27,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:29,079 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:29,441 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:30,961 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:30,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:32,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:33,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:33,442 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:34,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:34,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:34,315 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:35,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:25:35,106 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:25:35,106 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:25:35,106 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:25:35,443 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:25:36,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:36,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:37,444 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:38,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:38,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:40,107 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:40,940 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:40,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:42,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:42,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:43,446 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:45,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:45,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:46,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:47,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:47,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:49,125 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:25:49,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:25:49,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:25:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:25:49,448 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:49,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:49,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:51,296 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:52,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:52,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:55,450 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:25:56,348 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:25:56,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:56,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:57,229 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:25:58,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:25:58,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:01,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:01,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:01,453 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:02,297 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:03,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:03,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:04,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:04,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:04,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:06,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:06,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:07,381 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:07,455 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:08,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:08,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:11,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:11,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:12,456 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:13,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:13,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:13,459 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:16,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:16,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:17,457 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:18,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:18,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:19,125 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:26:19,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:19,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:19,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:19,466 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:20,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:20,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:23,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:25,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:25,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:25,469 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:27,234 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:27,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:28,898 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:29,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:30,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:31,471 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:33,730 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:26:33,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:33,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:33,975 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:34,128 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:34,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:35,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:35,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:37,474 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:38,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:38,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:39,030 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:40,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:40,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:43,476 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:43,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:43,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:44,097 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:45,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:45,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:48,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:48,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:49,128 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:26:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:26:49,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:26:49,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:26:49,478 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:50,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:50,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:53,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:53,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:54,227 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:26:55,480 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:26:55,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:55,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:57,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:57,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:26:59,228 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:00,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:00,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:01,482 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:02,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:02,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:04,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:04,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:04,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:04,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:05,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:05,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:07,485 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:07,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:07,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:09,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:11,255 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:27:11,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:11,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:13,487 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:14,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:14,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:14,615 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:16,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:16,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:19,130 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:27:19,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:19,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:19,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:19,489 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:20,265 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:21,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:21,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:23,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:23,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:25,491 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:25,747 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:26,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:26,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:28,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:28,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:30,748 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:31,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:31,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:31,493 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:33,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:33,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:34,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:34,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:36,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:36,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:36,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:37,495 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:38,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:38,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:41,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:41,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:41,879 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:43,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:43,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:43,497 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:46,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:46,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:46,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:27:49,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:27:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:27:49,258 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:27:49,698 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:27:49,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:49,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:51,500 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:52,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:52,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:52,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:54,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:54,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:57,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:27:57,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:57,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:58,221 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:27:59,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:27:59,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:02,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:02,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:03,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:03,505 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:04,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:04,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:04,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:04,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:07,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:28:07,418 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:28:07,419 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:28:07,421 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:28:07,506 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:28:07,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:07,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:08,422 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:09,507 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:09,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:09,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:12,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:12,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:14,412 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:15,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:15,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:15,510 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:18,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:18,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:19,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:19,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:19,348 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:28:19,490 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:20,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:20,090 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:21,512 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:22,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:22,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:24,490 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:24,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:25,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:27,515 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:28,533 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:28:28,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:28,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:29,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:31,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:31,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:33,456 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:33,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:33,518 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:34,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:34,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:35,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:35,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:35,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:38,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:38,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:39,520 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:40,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:40,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:40,674 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:43,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:43,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:45,522 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:45,732 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:48,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:48,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:28:49,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:28:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:28:49,315 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:28:50,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:50,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:50,886 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:51,524 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:53,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:53,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:55,886 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:28:57,526 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:28:58,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:28:58,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:00,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:00,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:00,955 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:03,529 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:03,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:03,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:04,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:04,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:06,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:07,139 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:29:07,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:07,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:09,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:09,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:09,531 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:12,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:12,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:12,085 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:14,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:14,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:15,533 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:17,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:17,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:17,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:19,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:19,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:19,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:19,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:20,751 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:29:21,535 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:22,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:22,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:22,752 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:24,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:24,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:26,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:26,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:27,537 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:28,278 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:29,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:29,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:31,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:31,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:33,338 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:33,539 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:34,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:34,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:34,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:34,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:36,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:36,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:38,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:39,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:39,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:39,542 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:41,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:41,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:43,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:44,943 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:29:44,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:44,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:45,545 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:47,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:47,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:48,468 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:29:49,129 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:29:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:29:49,265 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:29:49,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:49,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:51,547 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:52,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:52,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:53,525 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:54,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:54,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:57,549 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:29:57,824 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:57,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:58,576 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:29:59,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:29:59,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:01,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:01,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:03,551 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:03,645 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:04,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:04,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:04,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:04,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:06,757 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:06,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:08,694 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:09,554 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:09,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:09,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:11,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:11,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:13,695 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:14,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:14,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:15,555 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:16,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:16,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:18,759 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:19,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:19,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:19,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:19,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:20,265 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:30:21,558 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:23,396 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:30:23,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:23,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:23,832 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:26,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:26,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:27,560 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:28,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:28,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:28,877 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:30,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:30,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:33,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:33,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:33,562 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:33,932 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:34,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:34,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:34,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:35,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:35,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:38,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:38,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:38,981 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:39,052 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:30:39,054 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:30:39,054 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:30:39,054 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:30:39,565 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:30:39,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:40,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:40,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:41,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:43,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:43,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:44,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:45,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:45,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:45,568 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:48,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:48,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:49,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:30:49,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:30:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:30:49,575 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:50,322 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:30:50,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:50,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:50,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:51,575 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:53,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:53,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:55,577 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:55,839 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:30:57,577 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:30:58,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:30:58,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:01,423 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:02,145 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:31:02,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:02,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:03,579 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:04,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:04,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:04,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:04,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:04,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:06,725 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:06,940 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:06,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:07,580 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:08,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:08,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:11,725 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:11,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:11,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:13,582 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:13,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:13,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:16,794 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:16,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:16,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:18,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:18,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:19,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:19,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:19,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:19,347 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:31:19,584 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:21,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:21,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:22,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:23,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:23,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:25,586 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:25,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:25,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:27,907 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:28,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:28,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:31,588 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:32,981 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:34,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:34,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:34,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:36,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:36,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:37,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:38,257 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:39,657 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:31:39,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:39,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:42,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:42,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:43,257 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:44,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:44,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:45,592 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:47,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:47,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:48,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:31:49,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:31:49,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:31:49,307 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:31:49,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:49,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:51,594 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:52,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:52,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:53,381 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:54,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:54,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:57,596 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:31:58,436 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:31:59,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:31:59,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:02,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:02,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:03,481 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:03,598 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:04,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:04,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:04,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:04,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:04,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:06,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:06,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:08,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:09,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:09,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:09,600 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:11,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:11,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:13,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:14,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:14,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:15,603 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:17,983 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:32:18,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:18,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:18,602 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:19,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:19,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:19,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:20,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:20,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:20,887 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:32:21,605 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:22,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:22,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:23,888 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:24,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:24,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:27,607 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:27,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:27,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:29,738 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:29,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:29,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:31,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:31,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:33,609 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:34,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:34,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:34,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:34,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:35,349 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:36,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:36,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:39,611 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:40,860 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:41,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:41,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:44,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:44,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:45,613 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:45,903 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:46,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:46,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:32:49,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:32:49,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:32:49,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:49,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:50,512 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:32:51,514 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:51,615 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:51,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:51,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:55,145 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:32:55,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:55,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:56,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:32:56,106 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:32:56,106 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:32:56,107 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:32:56,616 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:32:57,107 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:32:57,617 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:32:58,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:32:58,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:00,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:00,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:02,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:02,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:03,088 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:03,618 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:04,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:04,130 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:04,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:05,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:05,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:08,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:08,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:08,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:09,620 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:10,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:10,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:13,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:13,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:13,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:15,622 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:18,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:18,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:18,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:19,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:19,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:19,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:20,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:20,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:20,580 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:33:21,624 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:23,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:23,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:23,581 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:25,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:25,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:27,626 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:28,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:28,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:29,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:30,375 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:30,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:33,788 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:33:33,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:33,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:34,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:34,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:34,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:34,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:34,628 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:36,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:36,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:38,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:38,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:39,424 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:40,630 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:41,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:41,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:43,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:43,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:44,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:46,632 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:46,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:46,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:48,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:48,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:49,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:33:49,131 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:33:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:33:49,351 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:33:50,352 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:50,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:51,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:52,634 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:53,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:53,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:55,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:33:56,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:56,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:58,636 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:33:58,777 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:33:58,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:00,594 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:01,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:01,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:03,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:03,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:04,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:04,132 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:04,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:04,638 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:06,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:06,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:06,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:08,639 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:08,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:08,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:10,640 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:11,870 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:12,291 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:34:12,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:12,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:14,641 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:15,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:15,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:16,642 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:17,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:17,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:17,601 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:19,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:19,132 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:19,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:19,289 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:34:20,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:20,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:20,643 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:22,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:22,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:23,292 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:25,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:25,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:26,645 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:27,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:27,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:28,438 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:30,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:30,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:30,647 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:32,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:32,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:34,010 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:34,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:34,132 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:34,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:34,648 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:35,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:35,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:37,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:37,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:39,917 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:40,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:40,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:40,650 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:42,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:42,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:45,056 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:45,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:45,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:46,652 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:47,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:47,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:49,132 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:34:49,133 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:34:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:34:49,310 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:34:50,335 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:50,563 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:34:50,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:50,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:50,654 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:53,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:53,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:55,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:55,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:56,244 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:34:56,656 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:34:58,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:34:58,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:00,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:00,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:01,894 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:02,658 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:03,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:03,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:04,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:04,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:04,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:05,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:05,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:05,847 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:35:05,848 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:35:05,849 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:35:05,849 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:35:06,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:35:07,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:07,850 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:08,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:08,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:08,661 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:10,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:10,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:13,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:13,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:13,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:14,663 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:15,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:15,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:18,665 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:19,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:19,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:19,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:20,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:20,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:20,455 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:35:20,455 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:22,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:22,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:24,667 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:24,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:24,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:26,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:28,199 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:35:28,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:28,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:28,669 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:31,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:31,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:32,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:33,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:33,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:34,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:34,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:34,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:34,671 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:35,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:35,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:37,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:37,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:37,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:40,673 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:40,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:40,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:42,605 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:42,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:42,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:44,674 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:45,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:45,793 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:47,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:47,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:48,584 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:49,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:35:49,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:35:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:35:50,234 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:35:50,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:50,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:52,676 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:54,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:35:55,467 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:55,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:56,678 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:35:57,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:57,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:35:59,950 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:00,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:00,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:02,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:02,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:02,680 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:04,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:04,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:04,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:05,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:05,969 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:36:05,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:06,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:08,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:08,682 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:08,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:10,546 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:10,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:10,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:12,683 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:13,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:13,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:15,548 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:15,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:15,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:18,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:18,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:18,685 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:19,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:19,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:19,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:20,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:20,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:20,923 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:36:20,926 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:23,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:23,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:24,687 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:25,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:25,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:26,388 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:28,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:28,689 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:30,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:30,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:32,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:33,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:33,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:34,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:34,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:34,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:34,691 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:35,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:35,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:37,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:38,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:38,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:40,694 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:43,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:43,616 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:36:43,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:43,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:46,280 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:46,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:46,696 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:48,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:48,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:48,813 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:49,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:36:49,134 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:36:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:36:49,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:36:51,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:51,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:52,698 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:53,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:53,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:54,463 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:36:55,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:55,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:56,699 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:36:58,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:58,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:36:59,464 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:01,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:01,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:02,702 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:03,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:03,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:04,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:04,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:04,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:05,285 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:05,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:05,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:07,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:07,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:08,704 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:10,819 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:10,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:10,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:12,705 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:12,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:12,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:15,693 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:15,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:16,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:17,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:17,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:18,707 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:19,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:19,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:19,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:19,290 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:37:21,071 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:37:21,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:21,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:21,291 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:22,107 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:37:22,108 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:37:22,109 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:37:22,111 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:37:22,709 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:37:23,710 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:23,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:23,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:24,710 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:25,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:25,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:27,113 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:28,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:28,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:30,712 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:30,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:30,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:32,933 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:33,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:33,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:34,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:34,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:34,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:34,714 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:35,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:35,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,266 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:38,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:38,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:40,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:40,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:40,718 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:43,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:43,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:43,755 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:45,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:45,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:46,719 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:48,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:48,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:49,134 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:37:49,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:37:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:37:49,324 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:37:49,324 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:50,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:50,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:50,721 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:53,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:53,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:54,514 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:37:55,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:55,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:56,723 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:37:59,264 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:37:59,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:37:59,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:00,422 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:01,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:01,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:02,726 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:04,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:04,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:04,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:04,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:06,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:06,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:06,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:08,728 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:09,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:09,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:11,495 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:11,712 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:11,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:14,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:14,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:14,730 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:16,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:16,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:17,262 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:18,731 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:19,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:19,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:19,352 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:38:19,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:19,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:21,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:21,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:22,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:24,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:24,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:24,734 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:26,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:26,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:28,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:29,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:29,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:30,736 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:31,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:31,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:33,732 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:34,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:34,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:34,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:34,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:36,738 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:38,067 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:38:38,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:38,096 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:39,340 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:40,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:40,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:42,740 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:42,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:43,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:44,913 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:45,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:45,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:46,742 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:47,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:47,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:49,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:38:49,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:38:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:38:49,364 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:38:50,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:50,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:50,795 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:52,743 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:38:52,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:52,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:55,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:38:55,637 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:55,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:57,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:57,679 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:38:58,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:00,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:00,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:01,227 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:02,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:02,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:02,746 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:04,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:04,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:05,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:05,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:06,281 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:07,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:07,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:08,748 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:10,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:10,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:12,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:12,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:12,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:14,751 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:15,787 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:39:15,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:15,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:17,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:18,732 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:18,752 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:18,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:19,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:19,135 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:19,329 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:39:20,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:20,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:22,330 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:23,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:23,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:24,754 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:25,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:25,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:27,686 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:28,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:28,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:30,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:30,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:30,756 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:33,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:33,333 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:33,528 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:34,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:34,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:35,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:35,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:36,758 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:38,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:38,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:39,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:39,474 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:39:39,475 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:39:39,475 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:39:39,476 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:39:39,759 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:39:39,759 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:40,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:40,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:40,760 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:43,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:43,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:44,477 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:45,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:45,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:46,761 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:48,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:48,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:49,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:39:49,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:39:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:39:49,295 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:39:50,227 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:50,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:50,296 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:52,763 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:53,632 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:39:53,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:53,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:55,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:39:56,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:56,765 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:39:58,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:39:58,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:00,558 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:01,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:01,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:02,767 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:03,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:03,446 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:04,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:04,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:06,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:06,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:06,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:08,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:08,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:08,769 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:10,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:10,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:11,881 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:12,770 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:13,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:13,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:16,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:16,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:16,894 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:18,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:18,757 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:18,773 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:19,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:19,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:19,360 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:40:21,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:21,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:22,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:23,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:23,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:24,775 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:26,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:26,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:27,517 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:28,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:28,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:30,777 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:31,963 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:40:31,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:32,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:33,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:34,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:34,136 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:34,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:34,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:36,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:36,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:36,779 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:39,090 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:39,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:39,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:40,781 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:41,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:41,606 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:44,093 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:44,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:44,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:46,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:46,727 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:46,783 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:49,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:40:49,137 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:40:49,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:40:49,292 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:40:49,293 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:49,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:49,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:51,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:51,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:52,785 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:54,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:54,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:55,025 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:40:56,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:56,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:56,786 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:40:59,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:40:59,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:00,677 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:01,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:01,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:02,788 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:04,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:04,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:04,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:04,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:04,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:06,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:06,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:06,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:08,790 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:09,832 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:41:09,858 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:09,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:11,477 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:12,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:12,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:14,792 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:17,363 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:17,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:17,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:18,793 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:19,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:19,137 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:19,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:41:19,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:19,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:22,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:22,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:23,085 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:24,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:24,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:24,795 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:27,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:27,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:28,085 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:29,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:29,419 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:30,797 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:32,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:32,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:33,890 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:34,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:34,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:34,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:34,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:34,798 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:37,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:37,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:39,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:39,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:39,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:40,801 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:42,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:42,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:44,281 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:44,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:44,667 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:46,803 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:47,708 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:41:47,737 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:47,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:49,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:41:49,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:41:49,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:41:50,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:50,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:51,187 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:41:51,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:52,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:52,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:52,804 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:55,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:55,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:56,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:41:56,497 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:41:56,498 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:41:56,499 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:41:56,499 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:41:56,805 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:41:57,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:57,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:41:57,806 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:41:58,806 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:00,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:00,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:01,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:02,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:02,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:02,808 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:04,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:04,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:04,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:05,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:05,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:07,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:07,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:07,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:08,811 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:10,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:10,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:12,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:12,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:12,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:14,813 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:15,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:15,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:17,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:17,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:17,737 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:19,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:19,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:19,354 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:42:20,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:20,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:20,815 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:22,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:22,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:23,356 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:25,431 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:42:25,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:25,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:26,817 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:28,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:28,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:28,669 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:30,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:30,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:30,818 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:33,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:33,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:34,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:34,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:34,265 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:35,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:35,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:36,820 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:38,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:38,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:39,499 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:40,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:40,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:42,822 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:43,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:43,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:44,844 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:45,076 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:45,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,152 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:48,824 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:49,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:42:49,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:42:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:42:49,305 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:42:50,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:50,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:50,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:53,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:53,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:54,826 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:55,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:55,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:56,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:42:57,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:57,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:58,827 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:42:59,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:42:59,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:02,030 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:03,294 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:43:03,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:03,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:04,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:04,138 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:04,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:04,830 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:06,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:06,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:07,269 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:08,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:08,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:10,831 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:11,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:11,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:12,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:13,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:13,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:14,832 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:15,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:15,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:17,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:17,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:18,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:19,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:19,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:19,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:19,302 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:43:20,834 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:20,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:20,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:22,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:22,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:23,439 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:25,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:25,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:26,837 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:27,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:27,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:29,205 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:30,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:30,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:30,838 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:32,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:32,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:34,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:34,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:34,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:34,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:35,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:35,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:36,841 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:37,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:37,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:39,938 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:41,122 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:43:41,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:41,156 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:42,843 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:43,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:43,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:45,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:45,884 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:45,935 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:48,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:48,837 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:48,846 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:49,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:43:49,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:43:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:43:49,342 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:43:50,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:50,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:51,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:52,847 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:53,841 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:53,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:55,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:55,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:56,615 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:43:58,850 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:43:58,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:43:58,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:00,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:00,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:02,564 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:03,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:03,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:04,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:04,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:04,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:04,852 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:05,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:05,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:08,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:08,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:09,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:10,854 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:12,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:12,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:13,467 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:14,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:14,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:44:14,366 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:44:14,366 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:44:14,367 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:44:14,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:14,855 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:44:16,856 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:17,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:17,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:19,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:19,228 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:19,229 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:19,418 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:44:19,419 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:20,790 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:44:20,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:20,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:20,858 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:23,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:23,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:24,419 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:25,765 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:25,777 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:26,860 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:28,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:28,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:30,017 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:30,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:30,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:32,862 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:33,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:33,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:34,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:34,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:34,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:35,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:35,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:35,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:36,864 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:38,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:38,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:40,749 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:40,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:40,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:42,865 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:43,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:43,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:45,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:45,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:46,663 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:48,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:48,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:48,868 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:49,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:44:49,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:44:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:44:49,272 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:44:50,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:50,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:52,274 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:52,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:52,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:54,871 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:44:55,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:55,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:57,461 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:44:59,181 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:44:59,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:44:59,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:00,873 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:02,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:02,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:03,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:04,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:04,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:04,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:04,227 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:04,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:04,874 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:07,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:07,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:08,372 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:09,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:09,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:10,876 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:11,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:11,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:13,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:13,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:14,080 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:15,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:15,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:16,878 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:18,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:18,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:19,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:19,139 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:19,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:20,293 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:45:20,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:20,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:20,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:20,880 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:23,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:23,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:25,295 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:25,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:25,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:26,882 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:28,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:28,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:30,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:30,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:30,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:32,884 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:33,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:33,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:34,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:34,140 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:34,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:35,902 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:36,886 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:37,046 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:45:37,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:37,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:40,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:40,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:40,903 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:42,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:42,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:42,888 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:45,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:45,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:46,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:47,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:47,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:48,890 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:49,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:45:49,140 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:45:49,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:45:49,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:49,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:50,914 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:45:51,915 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:51,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:51,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:52,892 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:45:54,737 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:54,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:56,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:56,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:56,919 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:45:59,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:59,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:45:59,894 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:01,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:01,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:01,920 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:04,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:04,140 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:04,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:04,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:04,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:06,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:06,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:07,273 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:09,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:09,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:11,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:11,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:12,274 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:14,979 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:46:15,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:15,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:15,899 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:17,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:17,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:18,228 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:19,139 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:19,140 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:19,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:19,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:19,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:20,560 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:46:22,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:22,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:23,561 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:23,902 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:24,742 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:24,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:27,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:27,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:27,903 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:28,641 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:29,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:29,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:32,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:32,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:33,906 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:34,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:34,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:34,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:34,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:34,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:34,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:37,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:37,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:37,907 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:39,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:39,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:39,831 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:41,096 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:46:41,097 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:46:41,097 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:46:41,098 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:46:41,909 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:46:42,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:42,235 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:42,909 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:43,910 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:44,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:44,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:45,099 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:47,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:47,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:49,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:46:49,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:46:49,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:46:49,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:49,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:49,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:46:49,912 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:50,378 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:52,611 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:46:52,643 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:52,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:53,913 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:46:55,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:55,712 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:55,761 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:46:57,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:57,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:46:59,915 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:01,675 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:02,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:02,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:04,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:04,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:04,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:04,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:04,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:05,917 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:07,289 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:07,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:07,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:09,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:09,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:09,918 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:12,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:12,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:12,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:14,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:14,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:15,920 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:17,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:17,875 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:18,333 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:19,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:19,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:19,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:19,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:20,607 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:47:21,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:21,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:23,609 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:23,923 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:23,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:23,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:27,925 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:29,131 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:30,895 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:47:30,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:30,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:32,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:32,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:33,927 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:34,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:34,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:34,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:34,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:35,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:35,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:37,928 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:39,954 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:40,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:40,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:42,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:42,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:43,930 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:44,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:44,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:45,852 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:47,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:47,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:49,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:47:49,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:47:49,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:47:49,933 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:50,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:50,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:50,698 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:47:51,699 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:52,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:52,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:54,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:54,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:55,935 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:47:57,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:57,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:57,629 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:47:59,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:59,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:47:59,936 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:01,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:01,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:03,437 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:04,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:04,141 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:04,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:04,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:04,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:05,937 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:08,448 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:48:08,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:08,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:09,292 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:10,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:10,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:11,940 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:13,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:13,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:14,326 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:15,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:15,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:15,941 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:17,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:17,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:19,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:19,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:19,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:19,324 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:48:20,235 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:20,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:20,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:21,943 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:22,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:22,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:24,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:25,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:25,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:27,945 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:29,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:29,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:31,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:31,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:31,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:33,947 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:34,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:34,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:34,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:34,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:34,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:36,369 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:36,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:36,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:38,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:39,949 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:41,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:41,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:42,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:43,950 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:45,464 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:48:45,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:45,492 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:47,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:48,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:48,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:49,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:48:49,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:48:49,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:48:49,356 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:48:49,952 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:50,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:50,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:52,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:53,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:53,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:55,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:55,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:55,954 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:48:57,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:57,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:48:57,746 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:48:58,582 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:48:58,584 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:48:58,584 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:48:58,585 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:48:58,955 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:48:59,956 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:00,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:00,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:00,956 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:02,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:02,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:03,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:04,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:04,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:04,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:04,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:04,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:05,958 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:07,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:07,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:09,448 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:09,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:09,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:11,960 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:12,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:12,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:14,732 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:17,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:17,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:17,961 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:19,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:19,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:19,336 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:49:19,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:19,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:20,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:23,103 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:49:23,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:23,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:23,964 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:26,115 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:26,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:26,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:27,965 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:28,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:28,481 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:31,116 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:31,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:31,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:33,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:33,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:33,966 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:34,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:34,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:34,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:36,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:36,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:36,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:38,381 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:38,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:39,968 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:41,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:41,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:41,482 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:43,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:43,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:45,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:45,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:45,970 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:47,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:48,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:48,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:49,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:49:49,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:49:49,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:49:49,972 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:50,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:50,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:50,642 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:49:52,643 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:49:52,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:52,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:55,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:55,709 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:55,974 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:49:57,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:57,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:49:57,944 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:01,127 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:50:01,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:01,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:01,976 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:03,867 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:04,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:04,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:04,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:04,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:06,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:06,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:07,978 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:08,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:08,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:09,230 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:11,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:11,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:11,980 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:13,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:13,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:14,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:15,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:15,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:17,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:17,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:17,982 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:19,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:19,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:20,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:20,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:20,510 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:50:20,512 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:22,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:22,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:23,986 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:25,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:25,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:26,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:27,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:27,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:29,989 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:30,523 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:30,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:32,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:32,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:32,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:34,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:34,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:34,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:34,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:34,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:35,991 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:38,323 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:50:38,334 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:38,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:38,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:41,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:41,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:41,993 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:43,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:43,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:43,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:46,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:46,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:47,995 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:48,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:48,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:49,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:50:49,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:50:49,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:50:50,804 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:50:50,804 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:51,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:51,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:53,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:53,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:53,998 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:56,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:56,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:56,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:50:57,999 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:50:58,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:50:58,208 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:01,809 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:03,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:03,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:04,001 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:04,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:04,142 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:04,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:06,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:06,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:07,243 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:08,002 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:08,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:08,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:11,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:11,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:13,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:13,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:13,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:14,004 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:16,710 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:51:16,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:16,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:16,920 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:51:16,921 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:51:16,921 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:51:16,921 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:51:17,006 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:51:18,922 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:19,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:19,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:19,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:19,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:19,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:19,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:20,007 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:20,604 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:51:21,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:21,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:24,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:24,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:24,816 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:26,009 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:26,338 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:26,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:29,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:29,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:30,010 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:30,635 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:31,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:31,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:34,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:34,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:34,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:34,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:34,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:36,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:36,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:36,349 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:36,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:39,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:39,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:41,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:41,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:41,692 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:42,015 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:44,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:44,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:46,016 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:46,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:46,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:46,760 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:49,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:51:49,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:51:49,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:51:49,227 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:49,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:49,267 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:51:51,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:51,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:52,018 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:52,702 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:54,751 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:51:54,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:54,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:57,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:57,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:58,020 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:51:58,384 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:51:59,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:51:59,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:02,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:02,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:03,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:04,022 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:04,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:04,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:04,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:04,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:04,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:07,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:07,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,470 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:09,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:09,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:10,024 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:12,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:12,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:14,026 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:14,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:14,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:14,692 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:17,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:17,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:19,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:19,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:19,330 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:52:19,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:19,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:20,029 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:20,331 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:22,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:22,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:24,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:24,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:26,031 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:26,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:27,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:27,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:29,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:29,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:31,198 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:32,033 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:32,871 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:52:32,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:32,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:34,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:34,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:34,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:35,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:35,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:37,125 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:37,762 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:37,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:38,035 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:40,767 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:40,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:42,036 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:42,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:42,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:43,034 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:45,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:45,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:47,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:47,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:48,038 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:48,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:49,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:52:49,143 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:52:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:52:49,286 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:52:50,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:50,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:52,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:52,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:54,041 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:52:54,769 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:52:55,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:55,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:57,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:52:57,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:00,043 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:00,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:00,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:00,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:02,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:02,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:04,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:04,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:04,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:05,363 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:05,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:05,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:06,045 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:07,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:07,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:10,046 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:10,698 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:11,111 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:53:11,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:11,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:13,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:13,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:16,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:16,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:16,048 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:16,638 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:18,954 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:18,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:19,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:19,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:19,314 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:53:20,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:21,011 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:22,050 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:22,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:23,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:23,989 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:26,028 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:26,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:26,051 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:28,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:28,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:28,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:30,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:30,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:32,054 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:33,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:33,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:34,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:34,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:34,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:34,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:35,298 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:53:35,299 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:53:35,299 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:53:35,300 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:53:35,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:35,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:36,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:53:37,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:38,055 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:38,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:38,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:39,301 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:40,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:40,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:43,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:43,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:44,057 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:45,069 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:45,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:45,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:48,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:49,071 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:53:49,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:49,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:49,143 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:53:49,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:53:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:53:49,371 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:53:50,372 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:51,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:51,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:53,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:53,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:54,060 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:53:55,869 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:53:56,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:56,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:58,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:53:58,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:00,062 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:01,769 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:01,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:01,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:03,874 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:03,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:04,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:04,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:04,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:06,064 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:06,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:06,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:07,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:08,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:08,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:11,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:11,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:12,067 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:13,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:13,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:13,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:16,068 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:16,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:16,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:18,660 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:18,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:18,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:19,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:19,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:19,319 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:54:21,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:21,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:22,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:23,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:23,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:23,874 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:26,873 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:54:26,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:26,908 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:28,071 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:29,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:29,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:29,771 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:31,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:31,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:32,072 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:34,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:34,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:34,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:34,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:34,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:35,324 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:36,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:36,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:38,074 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:39,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:39,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:40,726 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:41,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:41,506 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:44,075 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:44,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:44,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:46,261 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:46,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:46,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:49,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:54:49,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:54:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:54:49,286 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:54:49,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:49,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:50,078 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:51,287 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:51,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:51,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:54,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:54,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,079 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:54:56,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:56,453 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:54:59,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:54:59,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:00,081 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:01,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:01,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:02,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:04,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:04,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:04,720 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:55:04,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:04,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:06,086 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:07,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:07,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:07,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:09,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:12,087 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:12,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:12,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:12,878 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:14,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:14,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:17,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:17,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:18,090 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:18,226 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:19,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:19,144 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:19,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:19,291 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:55:19,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:19,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:22,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:22,280 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:22,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,113 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:24,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:24,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:27,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:27,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:28,092 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:29,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:29,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:30,062 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:32,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:32,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:34,094 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:34,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:34,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:34,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:34,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:35,300 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:36,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:36,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:38,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:39,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:40,097 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:40,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:42,366 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:55:42,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:42,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:44,098 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:45,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:45,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:45,814 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:47,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:47,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:49,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:55:49,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:55:49,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:55:49,363 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:55:49,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:49,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:50,100 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:51,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:51,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:51,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:53,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:55:53,844 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:55:53,844 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:55:53,844 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:55:54,101 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:55:54,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:54,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:55,101 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:56,101 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:55:56,845 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:55:56,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:56,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:55:59,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:00,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:02,103 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:02,659 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:03,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:03,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:04,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:04,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:05,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:06,105 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:07,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:07,939 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:10,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:10,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:12,106 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:13,424 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:13,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:13,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:15,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:15,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:18,108 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:18,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:18,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:19,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:19,236 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:19,237 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:19,373 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:56:19,374 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:21,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:21,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:22,109 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:25,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:25,524 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:56:25,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:25,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:28,112 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:28,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:28,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:30,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:30,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:31,002 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:33,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:33,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:34,114 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:34,144 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:34,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:35,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:35,731 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:36,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:38,115 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:38,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:38,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:40,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:40,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:41,543 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:43,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:43,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:44,117 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:45,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:45,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:47,225 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:48,118 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:48,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:48,541 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:49,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:56:49,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:56:49,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:56:50,120 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:50,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:50,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:50,769 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:56:52,685 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:53,120 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:53,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:53,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:55,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:55,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:57,122 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:56:58,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:56:58,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:58,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:56:59,123 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:01,124 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:02,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:02,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:03,610 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:04,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:04,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:05,125 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:05,869 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:57:05,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:05,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:08,636 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:08,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:08,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:09,126 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:10,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:10,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:13,636 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:13,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:13,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:15,128 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:15,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:15,889 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:17,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:17,928 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:18,674 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:19,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:19,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:19,317 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:57:20,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:20,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:21,130 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:22,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:22,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:23,727 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:24,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:24,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:27,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:27,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:27,133 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:28,776 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:29,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:29,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:32,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:32,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:33,134 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:33,812 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:34,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:34,145 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:35,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:35,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:37,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:37,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:38,879 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:39,137 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:40,161 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:40,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:43,757 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:57:43,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:43,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:43,880 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:45,138 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:45,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:45,820 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:48,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:48,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:48,931 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:49,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:57:49,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:57:49,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:57:49,347 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:57:50,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:50,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:51,140 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:52,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:52,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:53,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:55,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:55,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:56,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 19:57:56,127 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 19:57:56,127 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 19:57:56,128 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 19:57:56,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 19:57:57,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:57,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:57,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:57:58,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:57:59,129 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:57:59,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:00,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:02,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:02,247 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:03,144 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:04,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:04,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:04,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:04,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:04,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:07,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:07,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:09,146 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:09,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:09,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:10,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:12,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:12,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:14,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:14,364 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:15,148 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:15,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:17,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:17,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:19,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:19,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:19,343 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:58:20,251 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:21,021 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:58:21,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:21,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:21,150 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:23,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:23,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:25,252 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:26,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:26,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:27,153 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:28,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:28,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:30,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:30,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:30,308 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:33,154 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:33,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:33,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:34,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:34,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:35,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:35,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:35,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:37,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:37,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:40,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:40,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:40,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:41,157 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:42,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:42,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:45,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:45,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:45,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:47,159 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:47,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:47,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:49,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:58:49,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:58:49,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:58:49,299 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:58:50,538 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:50,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:50,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:52,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:52,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:53,161 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:58:54,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:54,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:55,539 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:58:58,413 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:58:58,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:58,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:58:59,163 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:00,612 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:01,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:01,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:03,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:03,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:04,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:59:04,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:59:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:59:05,165 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:06,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:06,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:06,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:08,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:08,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:11,167 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:11,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:11,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:11,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:13,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:13,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:16,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:16,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:16,810 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:17,169 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:18,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:18,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:19,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:59:19,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:59:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:59:19,345 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:59:21,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:21,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:21,863 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:23,171 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:23,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:23,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:26,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:26,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:26,864 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:28,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:28,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:29,173 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:31,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:31,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:31,911 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:33,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:33,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:34,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:59:34,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:59:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:59:35,175 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:37,253 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 19:59:37,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:37,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:37,382 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:40,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:40,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:41,177 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:42,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:42,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:43,045 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:44,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:44,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:46,965 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:46,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:47,179 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:48,103 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:49,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 19:59:49,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 19:59:49,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 19:59:49,293 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 19:59:49,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:49,999 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:52,019 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:52,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:53,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:53,181 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:55,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:55,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:57,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:57,076 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 19:59:58,210 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 19:59:59,183 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 19:59:59,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:00,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:02,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:02,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:03,211 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:04,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:00:04,146 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:00:04,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:00:05,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:05,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:05,185 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:07,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:07,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:08,703 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:09,187 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:10,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:10,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:11,187 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:12,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:12,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:14,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:15,189 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:15,694 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:00:15,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:00:15,945 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:00:15,945 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:00:15,946 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:00:16,189 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:00:16,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:16,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:17,190 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:18,190 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:18,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:18,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:19,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:00:19,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:00:19,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:00:21,090 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:00:21,091 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:21,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:21,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:23,191 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:23,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:23,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:26,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:26,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:26,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:27,193 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:29,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:29,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:31,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:31,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:31,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:33,194 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:34,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:00:34,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:00:34,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:00:34,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:34,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:36,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:36,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:36,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:38,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:38,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:39,197 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:41,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:41,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:42,251 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:43,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:43,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:45,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:46,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:46,218 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:47,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:48,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:48,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:49,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:00:49,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:00:49,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:00:50,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:50,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:50,935 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:00:52,936 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:53,202 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:53,999 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:00:54,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:54,040 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:57,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:57,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:57,203 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:00:58,445 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:00:59,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:00:59,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:01,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:01,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:03,205 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:03,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:04,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:01:04,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:01:04,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:01:04,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:04,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:06,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:06,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:08,585 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:09,207 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:09,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:09,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:11,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:11,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:13,586 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:14,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:14,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:15,209 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:16,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:16,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:18,659 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:19,146 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:01:19,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:01:19,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:01:19,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:19,659 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:20,440 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:01:21,211 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:21,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:21,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:24,442 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:24,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:24,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:26,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:26,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:27,213 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:28,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:28,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:29,797 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:32,116 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:01:32,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:32,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:33,215 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:34,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:01:34,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:01:34,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:01:34,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:34,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:35,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:36,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:36,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:39,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:39,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:39,217 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:40,933 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:41,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:41,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:43,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:43,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:45,219 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:45,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:45,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:46,001 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:48,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:48,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:49,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:01:49,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:01:49,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:01:49,288 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:01:50,921 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:50,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:51,069 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:51,221 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:53,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:53,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:55,953 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:55,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:56,070 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:01:57,222 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:01:57,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:01:58,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:01,133 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:01,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:01,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:03,224 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:03,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:03,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:04,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:02:04,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:02:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:02:05,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:05,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:06,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:08,739 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:02:08,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:08,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:09,226 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:11,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:11,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:12,275 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:13,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:13,624 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:15,229 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:15,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:15,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:17,341 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:18,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:18,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:19,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:02:19,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:02:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:02:19,264 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:02:20,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:20,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:21,230 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:22,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:22,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:22,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:24,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:24,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:26,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:26,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:27,232 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:27,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:28,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:28,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:31,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:31,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:32,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:33,235 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:33,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:33,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:34,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:02:34,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:02:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:02:36,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:36,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:38,330 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:38,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:38,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:39,237 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:41,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:41,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:43,633 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:44,785 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:02:44,786 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:02:44,786 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:02:44,788 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:02:45,240 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:02:45,633 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:02:45,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:45,696 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:46,241 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:47,241 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:47,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:47,732 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:48,789 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:49,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:02:49,147 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:02:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:02:49,755 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:49,766 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:50,723 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:02:53,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:53,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:53,243 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:02:53,792 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:55,069 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:55,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:58,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:58,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:02:58,896 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:02:59,245 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:00,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:00,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:03,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:03,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:03,961 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:04,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:03:04,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:03:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:03:05,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:05,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:05,247 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:08,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:08,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:09,036 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:10,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:10,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:11,249 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:12,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:12,495 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:14,036 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:14,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:14,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:17,251 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:17,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:17,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:19,130 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:19,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:03:19,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:03:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:03:19,316 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:03:19,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:19,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:23,161 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:03:23,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:23,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:23,253 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:24,214 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:26,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:26,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:28,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:28,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:29,255 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:29,301 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:31,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:31,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:33,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:33,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:34,147 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:03:34,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:03:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:03:34,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:35,258 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:35,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:35,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:37,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:37,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:39,355 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:39,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:39,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:41,260 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:42,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:42,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:44,467 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:44,679 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:44,688 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:47,262 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:47,560 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:47,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:49,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:03:49,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:03:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:03:49,331 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:03:49,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:49,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:50,332 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:52,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:52,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:53,264 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:03:54,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:54,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:55,637 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:03:56,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:56,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:03:59,267 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:00,560 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:04:00,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:00,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:00,719 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:03,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:03,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:04,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:04:04,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:04:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:04:05,269 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:05,751 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:05,774 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:06,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:07,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:07,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:10,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:10,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:11,272 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:11,909 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:12,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:12,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:15,800 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:15,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:17,008 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:17,274 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:17,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:17,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:19,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:04:19,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:04:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:04:19,369 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:04:20,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:20,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:22,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:22,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:22,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:23,276 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:24,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:25,003 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:27,121 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:28,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:28,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:29,278 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:30,266 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:30,280 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:32,220 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:32,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:32,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:34,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:04:34,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:04:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:04:35,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:35,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:35,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:37,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:39,094 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:04:39,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:39,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:41,283 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:41,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:41,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:42,435 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:44,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:44,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:46,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:46,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:47,285 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:47,527 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:48,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:48,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:49,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:04:49,148 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:04:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:04:50,417 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:04:51,327 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:51,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:52,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:53,287 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:04:53,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:53,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:55,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:55,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:57,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:04:58,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:58,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:04:59,289 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:00,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:00,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:02,749 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:03,517 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:03,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:04,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:05:04,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:05:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:05:05,292 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:05,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:05,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:07,840 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:08,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:08,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:11,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:11,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:12,948 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:13,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:13,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:13,295 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:16,902 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:05:16,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:16,946 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:17,119 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:05:17,120 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:05:17,120 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:05:17,122 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:05:17,296 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:05:18,122 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:18,297 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:18,969 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:18,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:19,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:05:19,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:05:19,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:05:19,297 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:19,344 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:05:21,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:22,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:23,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:24,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:24,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:25,299 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:27,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:27,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:28,142 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:29,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:29,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:31,301 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:32,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:32,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:33,265 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:34,148 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:05:34,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:05:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:05:34,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:34,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:36,235 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:36,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:37,303 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:38,355 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:39,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:39,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:41,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:41,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:43,305 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:43,446 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:43,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:43,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:46,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:46,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:48,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:48,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:48,556 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:49,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:05:49,149 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:05:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:05:49,278 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:05:49,307 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:51,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:51,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:53,641 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:55,207 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:05:55,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:55,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:55,309 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:05:57,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:57,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:58,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:05:59,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:05:59,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:01,311 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:01,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:01,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:03,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:03,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:03,728 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:04,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:06:04,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:06:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:06:05,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:05,463 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:07,313 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:07,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:07,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:08,823 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:10,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:10,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:12,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:12,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:13,315 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:13,907 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:15,520 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:15,541 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:17,562 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:17,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:18,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:19,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:06:19,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:06:19,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:06:19,317 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:20,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:20,674 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:06:20,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:22,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:22,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:24,080 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:24,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:24,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:25,319 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:27,998 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:28,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:29,082 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:31,320 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:31,842 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:06:31,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:31,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:33,321 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:33,902 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:33,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:34,149 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:06:34,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:06:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:06:34,305 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:36,767 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:36,790 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:37,323 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:38,810 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:38,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:39,324 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:40,109 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:41,324 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:06:41,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:41,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:43,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:43,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:45,603 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:45,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:45,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:48,637 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:48,660 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:49,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:06:49,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:06:49,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:06:49,282 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:06:50,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:50,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:51,283 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:53,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:53,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:56,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:56,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:56,284 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:06:59,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:59,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:06:59,331 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:01,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:01,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:01,472 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:04,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:07:04,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:07:04,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:07:04,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:04,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:05,333 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:06,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:06,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:07,350 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:10,114 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:07:10,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:10,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:11,335 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:12,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:13,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:13,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:15,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:15,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:17,337 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:17,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:17,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:17,753 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:19,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:07:19,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:07:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:07:19,374 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:07:19,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:19,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:22,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:22,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:22,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:23,340 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:24,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:24,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:27,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:27,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:27,842 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:29,342 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:29,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:29,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:32,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:32,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:32,926 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:34,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:07:34,150 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:07:34,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:07:34,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:34,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:35,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:36,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:36,682 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:38,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:38,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:39,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:41,346 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:41,804 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:41,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:43,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:43,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:44,123 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:46,281 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:07:46,283 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:07:46,283 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:07:46,283 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:07:46,348 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:07:47,348 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:47,468 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:07:47,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:47,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:48,349 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:49,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:07:49,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:07:49,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:07:49,345 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:07:49,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:50,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:50,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:52,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:52,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:53,350 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:55,305 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:07:55,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:55,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:57,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:57,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:59,352 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:07:59,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:07:59,732 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:00,395 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:02,918 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:02,932 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:04,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:08:04,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:08:04,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:08:04,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:04,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:05,354 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:05,488 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:07,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:07,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:09,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:09,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:10,489 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:11,357 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:12,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:12,031 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:14,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:15,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:15,784 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:17,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:17,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:17,359 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:19,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:08:19,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:08:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:08:19,345 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:08:20,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:20,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:21,346 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:22,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:22,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:23,361 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:26,119 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:08:26,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:26,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:26,955 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:28,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:28,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:30,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:30,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:31,364 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:32,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:33,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:33,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:34,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:08:34,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:08:34,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:08:35,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:35,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:37,366 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:37,497 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:38,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:38,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:40,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:40,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:42,580 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:43,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:43,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:43,368 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:45,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:45,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:47,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:47,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:47,580 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:49,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:08:49,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:08:49,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:08:49,370 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:49,373 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:08:50,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:50,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:52,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:52,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:53,374 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:54,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:54,669 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:55,372 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:08:57,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:57,490 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:58,774 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:08:59,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:08:59,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:01,374 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:03,154 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:09:03,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:03,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:03,866 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:04,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:09:04,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:09:04,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:09:06,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:06,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:07,377 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:08,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:08,105 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:08,946 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:10,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:10,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:13,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:13,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:13,379 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:14,052 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:15,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:15,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:17,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:17,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:19,145 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:19,150 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:09:19,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:09:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:09:19,370 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:09:19,381 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:19,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:19,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:22,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:22,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:24,244 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:24,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:24,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:25,384 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:27,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:27,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:29,245 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:29,505 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:29,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:31,386 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:32,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:32,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:34,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:09:34,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:09:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:09:34,316 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:34,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:34,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:36,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:36,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:37,388 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:39,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:40,769 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:09:40,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:40,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:43,390 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:43,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:43,926 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:44,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:45,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:45,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:48,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:49,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:49,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:09:49,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:09:49,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:09:49,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:09:49,393 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:50,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:51,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:51,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:53,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:53,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:55,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:09:55,720 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:09:56,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:56,157 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:58,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:09:58,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:00,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:00,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:00,721 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:01,397 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:03,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:03,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:04,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:10:04,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:10:04,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:10:05,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:05,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:06,310 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:07,399 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:07,513 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:07,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:10,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:10,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:11,928 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:12,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:12,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:13,402 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:15,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:15,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:17,018 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:19,141 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:10:19,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:10:19,151 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:10:19,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:19,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:10:19,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:19,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:10:19,315 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:10:19,315 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:10:19,316 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:10:19,316 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:10:19,404 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:10:19,404 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:20,404 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:21,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:21,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:21,404 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:22,318 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:23,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:23,257 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:25,406 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:26,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:26,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:28,207 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:28,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:28,590 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:30,621 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:30,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:33,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:33,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:33,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:34,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:10:34,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:10:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:10:34,409 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:35,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:35,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:38,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:38,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:38,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:40,411 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:40,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:40,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:43,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:43,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:43,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:45,791 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:45,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:46,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:47,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:47,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:48,488 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:49,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:10:49,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:10:49,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:10:50,581 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:10:51,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:51,085 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:52,415 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:53,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:53,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:53,583 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:56,825 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:10:56,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:56,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:58,418 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:10:58,700 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:10:59,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:10:59,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:01,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:01,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:03,794 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:04,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:11:04,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:11:04,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:11:04,419 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:04,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:04,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:06,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:06,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:08,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:08,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:08,896 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:10,421 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:11,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:11,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:13,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:13,981 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:13,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:16,032 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:16,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:16,424 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:18,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:19,151 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:11:19,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:11:19,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:11:19,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:19,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:19,320 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:11:21,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:21,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:22,426 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:24,096 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:24,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:24,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:26,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:26,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:28,428 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:29,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:29,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:29,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:31,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:31,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:11:34,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:11:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:11:34,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:34,430 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:35,001 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:11:35,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:35,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:38,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:38,172 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:39,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:40,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:40,229 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:40,432 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:42,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:42,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:44,408 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:45,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:45,203 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:46,434 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:47,227 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:47,244 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:49,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:11:49,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:11:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:11:49,359 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:11:50,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:50,130 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:50,360 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:52,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:52,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:52,436 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:55,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:55,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:55,600 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:11:57,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:57,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:58,438 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:11:59,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:11:59,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:00,709 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:02,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:02,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:04,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:12:04,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:12:04,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:12:04,441 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:04,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:04,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:06,335 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:06,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:06,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:09,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:09,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:10,443 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:11,905 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:13,031 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:12:13,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:13,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:16,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:16,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:16,445 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:17,010 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:18,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:18,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:19,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:12:19,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:12:19,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:12:19,351 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:12:20,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:20,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:22,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:22,447 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:23,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:23,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:25,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:25,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:27,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:27,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:27,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:30,450 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:30,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:30,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:32,384 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:32,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:32,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:12:34,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:12:34,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:12:35,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:35,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:36,452 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:37,707 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:37,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:38,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:40,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:40,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:42,454 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:42,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:42,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:43,571 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:44,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:46,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:48,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:48,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:48,456 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:48,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:49,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:12:49,152 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:12:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:12:49,382 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:12:51,634 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:12:51,636 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:12:51,636 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:12:51,637 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:12:51,854 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:12:51,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:51,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:52,457 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:12:52,457 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:53,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:53,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:54,458 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:54,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:12:56,459 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:56,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:56,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:58,459 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:12:58,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:12:58,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:00,035 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:01,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:01,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:02,461 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:03,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:03,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:04,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:13:04,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:13:04,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:13:04,461 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:05,313 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:06,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:06,914 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:08,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:08,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:10,463 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:11,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:11,036 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:11,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:13,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:13,073 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:15,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:15,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:16,442 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:16,465 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:18,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:18,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:19,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:13:19,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:13:19,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:13:19,354 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:13:20,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:20,297 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:21,537 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:22,467 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:23,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:23,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:25,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:25,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:26,649 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:28,469 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:28,823 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:13:28,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:28,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:31,649 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:32,029 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:32,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:34,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:34,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:34,152 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:13:34,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:13:34,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:13:34,470 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:36,126 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:36,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:37,370 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:38,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:38,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:40,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:40,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:40,472 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:42,830 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:43,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:43,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:45,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:45,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:46,474 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:47,933 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:48,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:48,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:13:49,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:13:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:13:50,351 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:13:50,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:50,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:52,476 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:53,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:13:53,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:53,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:55,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:55,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:58,479 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:13:58,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:58,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:13:59,137 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:00,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:00,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:02,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:02,856 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:04,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:14:04,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:14:04,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:14:04,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:04,481 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:06,618 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:14:06,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:06,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:09,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:09,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:09,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:10,483 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:11,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:11,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:14,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:14,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:14,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:16,485 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:16,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:16,678 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:19,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:14:19,153 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:14:19,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:14:19,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:19,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:21,187 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:14:21,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:21,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:21,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:22,487 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:23,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:23,906 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:26,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:27,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:27,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:28,489 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:29,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:29,178 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:31,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:31,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:31,631 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:34,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:14:34,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:14:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:14:34,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:34,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:34,491 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:36,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:36,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:37,283 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:39,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:39,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:40,492 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:41,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:41,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:42,839 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:45,034 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:14:45,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:45,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:46,494 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:47,932 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:48,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:48,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:14:49,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:14:49,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:14:49,371 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:14:50,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:50,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:52,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:52,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:52,496 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:14:53,048 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:55,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:55,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:57,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:57,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:14:58,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:14:58,498 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:00,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:00,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:02,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:02,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:03,140 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:04,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:15:04,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:15:04,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:15:04,500 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:05,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:05,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:07,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:07,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:08,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:09,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:15:09,414 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:15:09,414 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:15:09,415 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:15:09,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:15:09,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:09,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:10,502 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:11,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:11,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:13,416 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:14,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:15,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:16,504 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:17,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:17,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:18,450 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:19,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:19,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:19,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:15:19,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:15:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:15:19,364 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:15:22,506 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:22,574 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:15:22,598 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:22,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:23,572 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:25,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:25,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:27,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:27,462 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:28,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:30,509 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:30,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:30,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:32,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:32,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:33,669 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:34,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:15:34,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:15:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:15:34,637 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:34,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:36,511 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:36,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:36,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:39,308 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:39,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:39,925 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:41,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:41,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:42,513 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:44,881 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:45,020 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:45,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:47,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:47,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:48,515 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:49,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:15:49,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:15:49,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:15:49,310 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:15:50,189 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:50,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:50,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:52,260 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:52,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:54,517 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:15:55,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:55,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:56,092 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:15:57,437 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:15:57,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:00,519 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:01,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:01,266 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:16:01,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:01,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:03,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:03,343 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:04,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:16:04,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:16:04,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:16:06,288 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:06,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:06,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:06,522 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:08,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:08,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:11,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:11,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:11,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:12,524 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:13,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:13,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:16,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:16,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:16,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:18,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:18,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:18,526 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:19,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:16:19,154 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:16:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:16:20,303 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:16:20,544 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:20,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:22,305 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:23,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:23,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:24,529 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:25,694 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:25,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:27,602 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:27,755 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:27,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:29,819 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:29,837 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:30,531 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:32,634 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:32,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:32,724 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:34,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:16:34,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:16:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:16:34,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:34,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:36,533 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:37,830 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:38,258 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:16:38,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:38,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:41,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:41,090 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:42,535 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:42,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:43,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:43,203 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:45,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:45,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:47,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:47,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:48,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:48,537 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:49,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:16:49,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:16:49,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:16:49,325 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:16:49,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:49,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:52,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:52,547 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:53,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:54,540 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:16:54,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:54,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:56,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:56,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:58,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:16:59,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:16:59,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:00,542 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:01,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:01,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:03,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:04,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:17:04,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:17:04,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:17:04,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:04,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:06,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:06,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:06,544 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:08,400 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:08,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:08,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:10,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:10,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:12,546 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:13,674 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:14,374 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:17:14,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:14,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:17,740 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:17,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:18,789 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:19,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:17:19,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:17:19,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:17:19,323 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:17:19,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:19,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:20,549 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:22,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:22,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:23,905 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:24,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:24,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:26,551 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:27,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:27,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:28,906 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:29,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:29,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:31,873 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:31,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:32,554 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:33,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:33,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:34,011 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:34,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:17:34,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:17:34,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:17:37,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:37,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:38,556 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:39,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:39,130 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:39,141 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:41,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:41,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:42,313 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:17:42,314 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:17:42,315 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:17:42,316 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:17:42,557 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:17:43,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:43,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:44,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:44,558 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:45,558 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:46,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:46,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:48,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:48,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:17:49,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:17:49,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:17:50,508 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:17:50,509 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:50,560 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:51,856 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:17:51,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:51,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:54,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:54,867 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:56,476 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:17:56,562 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:17:56,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:56,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:58,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:17:58,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:01,576 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:02,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:02,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:02,564 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:04,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:18:04,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:18:04,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:18:04,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:04,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:06,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:06,290 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:06,695 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:08,566 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:09,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:09,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:11,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:11,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:11,696 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:14,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:14,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:14,568 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:16,472 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:16,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:16,813 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:19,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:18:19,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:18:19,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:18:19,372 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:18:19,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:19,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:20,570 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:21,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:21,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:22,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:23,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:23,657 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:25,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:25,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:26,572 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:28,031 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:29,624 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:18:29,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:29,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:32,528 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:32,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:32,574 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:33,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:34,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:18:34,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:18:34,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:18:34,585 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:34,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:37,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:37,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:38,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:38,576 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:39,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:39,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:42,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:42,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:43,369 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:44,578 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:44,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:44,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:46,891 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:46,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:48,466 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:49,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:18:49,155 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:18:49,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:18:50,057 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:50,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:50,613 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:18:52,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:52,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:52,581 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:53,615 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:54,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:54,191 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:56,582 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:18:57,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:57,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:58,686 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:18:59,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:18:59,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:02,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:02,377 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:02,584 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:03,804 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:04,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:19:04,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:19:04,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:19:04,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:04,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:08,227 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:19:08,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:08,298 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:08,904 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:10,586 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:11,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:11,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:13,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:13,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:13,905 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:15,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:15,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:16,589 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:17,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:17,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:19,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:19:19,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:19:19,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:19:19,314 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:19,316 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:19:20,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:20,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:20,590 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:22,591 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:22,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:22,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:24,591 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:24,826 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:25,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:25,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:27,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:27,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:28,593 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:30,326 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:30,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:30,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:32,594 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:32,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:32,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:34,155 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:19:34,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:19:34,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:19:34,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:34,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:35,816 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:36,596 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:36,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:36,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:40,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:40,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:40,816 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:42,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:42,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:44,599 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:45,806 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:19:45,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:45,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:45,930 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:48,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:48,607 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:19:49,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:19:49,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:19:49,375 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:19:50,601 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:50,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:50,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:51,377 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:53,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:53,540 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:55,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:55,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:56,603 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:19:57,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:19:57,609 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:57,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:19:59,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:00,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:20:00,326 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:20:00,327 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:20:00,327 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:20:00,604 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:20:00,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:01,605 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:02,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:02,605 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:02,892 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:02,902 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:20:04,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:20:04,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:20:04,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:04,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:06,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:06,984 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:07,361 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:08,607 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:10,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:10,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:12,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:12,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:12,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:14,610 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:15,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:15,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:17,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:17,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:17,483 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:20:19,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:20:19,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:20:20,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:20,368 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:20:20,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:20,612 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:23,370 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:24,239 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:20:24,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:24,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:26,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:26,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:26,613 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:28,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:28,672 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:29,595 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:31,616 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:31,627 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:32,615 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:33,792 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:20:34,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:20:34,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:20:34,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:34,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:36,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:36,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:38,617 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:38,896 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:39,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:39,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:41,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:41,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:43,773 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:43,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:43,998 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:44,619 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:45,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:45,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:48,999 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:49,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:49,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:20:49,156 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:20:49,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:20:50,341 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:20:50,620 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:51,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:51,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:53,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:53,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:54,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:20:56,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:56,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:56,622 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:20:58,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:20:58,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:00,213 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:01,754 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:21:01,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:01,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:02,625 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:21:04,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:21:04,176 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:21:04,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:04,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:05,355 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:06,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:06,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:08,627 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:08,784 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:08,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:10,444 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:10,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:10,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:12,866 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:12,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:14,629 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:15,547 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:16,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:16,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:18,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:18,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:21:19,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:21:19,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:21:19,344 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:21:20,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:20,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:20,631 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:20,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:23,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:23,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:25,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:25,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:25,665 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:26,633 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:28,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:28,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:30,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:30,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:30,780 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:32,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:32,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:32,635 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:34,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:21:34,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:21:34,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:21:35,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:35,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:36,383 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:38,636 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:39,081 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:21:39,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:39,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:41,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:41,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:42,006 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:43,181 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:43,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:45,899 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:45,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:46,639 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:47,120 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:47,939 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:47,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:49,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:21:49,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:21:49,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:21:49,384 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:21:51,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:51,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:52,242 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:52,641 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:21:53,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:53,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:56,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:56,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:57,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:21:58,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:58,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:21:58,643 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:00,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:00,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:02,278 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:02,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:02,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:04,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:22:04,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:22:04,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:22:04,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:04,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:04,645 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:07,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:07,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:07,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:09,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:09,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:10,647 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:12,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:12,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:12,573 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:16,146 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:22:16,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:16,174 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:16,649 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:17,688 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:18,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:18,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:19,156 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:22:19,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:22:19,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:22:19,367 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:22:21,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:21,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:22,651 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:22,791 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:23,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:23,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:25,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:25,471 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:27,902 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:28,653 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:28,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:28,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:30,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:30,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:32,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:32,904 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:33,086 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:22:33,088 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:22:33,088 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:22:33,088 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:22:33,654 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:22:33,780 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:22:34,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:22:34,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:22:34,655 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:35,655 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:35,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:35,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:38,317 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:38,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:38,910 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:40,657 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:40,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:40,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:44,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:44,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:44,138 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:46,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:46,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:46,659 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:48,082 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:48,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:49,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:22:49,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:22:49,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:22:49,280 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:49,328 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:22:50,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:50,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:52,660 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:53,909 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:22:53,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:53,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:54,776 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:22:56,681 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:56,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:58,663 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:22:58,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:58,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:22:59,879 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:01,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:01,739 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:03,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:03,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:04,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:23:04,157 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:23:04,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:23:04,665 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:05,296 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:06,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:07,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:09,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:09,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:10,666 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:11,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:11,086 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:11,116 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:13,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:13,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:16,117 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:16,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:16,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:18,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:18,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:18,669 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:23:19,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:23:19,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:23:19,329 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:23:21,331 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:21,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:21,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:23,477 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:23,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:24,671 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:26,350 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:26,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:26,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:28,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:28,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:30,673 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:31,457 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:32,263 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:23:32,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:32,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:23:34,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:23:34,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:23:35,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:35,436 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:36,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:36,675 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:37,458 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:37,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:39,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:39,515 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:41,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:41,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:41,690 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:42,677 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:44,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:44,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:46,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:46,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:46,691 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:48,680 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:49,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:23:49,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:23:49,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:23:49,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:49,332 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:23:49,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:51,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:51,389 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:52,333 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:54,522 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:54,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:54,682 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:23:56,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:56,611 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:57,900 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:23:58,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:23:58,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:00,677 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:00,684 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:00,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:03,032 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:03,900 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:03,912 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:04,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:24:04,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:24:04,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:24:05,952 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:05,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:06,686 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:08,496 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:09,670 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:24:09,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:09,720 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:12,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:12,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:12,688 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:13,606 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:14,494 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:14,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:17,414 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:17,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:18,690 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:18,715 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:19,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:24:19,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:24:19,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:24:19,341 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:24:19,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:19,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:21,491 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:21,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:23,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:23,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:23,829 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:24,693 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:26,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:26,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:28,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:28,763 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:28,829 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:30,694 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:30,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:30,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:33,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:33,723 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:33,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:34,157 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:24:34,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:24:34,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:24:35,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:35,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:36,696 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:38,756 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:38,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:39,055 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:40,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:40,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:42,698 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:44,004 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:44,053 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:44,177 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:47,815 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:24:47,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:47,853 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:24:49,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:24:49,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:24:49,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:24:49,357 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:49,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:49,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:50,701 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:53,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:53,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:54,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:24:55,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:55,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:56,703 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:24:58,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:58,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:24:59,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:00,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:00,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:02,705 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:03,137 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:03,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:04,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:25:04,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:25:04,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:25:05,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:05,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:05,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:06,685 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:25:06,687 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:25:06,687 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:25:06,687 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:25:06,706 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:25:07,707 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:08,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:08,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:08,707 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:10,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:10,482 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:10,688 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:12,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:12,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:14,709 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:15,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:15,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:15,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:17,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:17,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:19,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:25:19,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:25:19,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:25:19,397 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:25:19,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:19,815 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:20,711 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:21,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:22,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:22,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:26,233 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:25:26,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:26,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:26,712 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:26,971 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:29,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:29,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:31,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:31,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:32,089 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:32,715 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:33,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:33,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:34,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:25:34,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:25:34,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:25:35,355 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:35,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:37,206 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:37,392 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:37,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:38,717 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:39,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:40,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:42,207 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:42,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:42,649 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:44,690 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:44,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:44,719 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:46,720 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:47,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:47,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:47,813 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:48,720 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:49,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:25:49,158 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:25:49,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:25:49,783 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:49,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:50,811 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:25:52,722 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:52,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:52,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:53,545 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:54,722 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:54,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:55,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:57,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:57,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:58,723 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:25:58,904 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:25:59,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:25:59,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:02,725 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:02,870 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:26:02,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:02,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:04,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:26:04,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:26:04,178 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:26:04,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:04,725 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:04,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:04,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:06,726 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:08,051 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:08,074 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:10,100 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:10,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:10,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:10,728 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:13,007 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:13,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:15,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:15,071 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:15,101 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:16,730 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:17,934 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:17,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:19,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:26:19,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:26:19,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:26:19,340 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:26:19,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:19,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:20,341 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:22,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:22,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:22,731 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:25,277 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:25,306 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:25,872 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:27,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:27,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:28,733 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:29,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:29,417 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:31,847 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:32,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:32,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:34,158 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:26:34,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:26:34,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:26:34,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:34,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:34,735 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:37,349 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:37,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:37,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:38,737 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:40,971 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:26:40,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:41,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:42,648 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:43,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:43,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:44,738 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:46,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:46,168 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:48,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:48,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:48,535 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:49,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:26:49,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:26:49,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:26:49,359 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:26:50,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:50,251 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:50,741 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:53,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:53,383 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:54,251 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:26:54,742 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:26:55,404 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:55,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:57,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:57,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:26:59,252 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:00,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:00,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:00,744 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:02,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:02,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:04,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:27:04,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:27:04,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:27:04,327 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:05,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:27:05,185 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:27:05,186 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:27:05,186 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:27:05,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:05,374 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:05,745 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:27:05,746 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:06,746 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:07,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:07,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:09,534 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:09,549 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:10,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:12,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:12,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:12,749 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:14,679 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:14,698 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:16,053 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:17,751 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:18,389 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:27:18,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:18,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:19,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:27:19,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:27:19,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:27:19,344 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:27:21,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:21,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:21,345 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:23,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:23,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:23,753 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:25,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:25,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:26,611 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:28,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:28,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:29,755 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:30,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:30,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:32,198 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:32,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:32,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:34,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:27:34,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:27:34,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:27:35,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:35,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:35,756 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:37,368 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:37,387 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:38,281 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:39,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:39,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:39,758 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:42,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:42,358 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:43,282 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:44,388 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:44,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:45,760 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:47,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:47,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:49,072 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:49,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:27:49,159 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:27:49,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:27:49,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:49,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:49,379 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:27:51,762 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:52,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:52,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:54,381 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:27:55,988 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:27:56,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:56,033 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:57,764 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:27:58,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:58,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:27:59,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:01,242 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:01,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:01,765 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:03,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:03,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:04,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:28:04,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:28:04,179 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:28:05,291 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:05,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:05,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:07,767 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:08,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:08,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:10,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:10,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:10,848 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:13,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:13,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:13,769 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:15,084 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:15,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:16,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:17,770 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:18,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:18,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:19,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:28:19,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:28:19,180 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:28:19,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:28:20,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:20,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:21,378 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:22,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:22,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:25,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:25,337 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:26,379 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:27,391 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:27,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:29,444 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:29,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:31,380 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:32,971 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:28:32,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:33,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:34,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:28:34,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:28:34,180 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:28:35,776 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:35,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:35,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:37,363 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:37,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:37,840 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:40,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:40,828 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:41,778 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:42,643 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:42,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:42,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:44,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:44,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:45,779 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:48,151 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:48,163 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:48,286 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:49,159 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:28:49,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:28:49,180 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:28:50,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:50,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:50,524 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:28:51,781 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:52,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:52,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:53,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:28:55,205 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:55,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:57,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:57,262 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:28:57,782 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:28:58,979 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:00,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:00,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:02,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:02,147 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:03,784 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:04,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:29:04,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:29:04,180 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:29:04,364 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:04,975 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:04,996 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:07,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:07,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:07,785 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:09,372 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:10,753 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:29:10,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:10,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:12,818 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:12,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:13,787 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:14,733 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:16,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:16,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:18,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:18,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:19,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:29:19,160 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:29:19,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:29:19,789 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:20,447 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:29:20,448 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:21,113 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:21,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:23,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:23,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:23,791 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:26,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:26,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:26,290 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:28,167 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:28,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:29,794 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:30,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:30,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:32,258 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:33,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:33,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:34,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:29:34,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:29:34,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:29:34,218 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:29:34,320 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:29:34,320 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:29:34,320 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:29:34,795 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:29:35,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:35,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:35,796 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:37,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:37,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:37,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:40,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:40,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:41,798 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:42,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:42,535 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:43,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:45,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:45,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:47,800 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:49,104 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:49,125 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:29:49,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:49,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:49,160 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:29:49,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:29:49,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:29:49,358 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:29:51,801 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:52,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:52,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:54,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:54,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:54,984 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:29:56,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:56,123 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:57,803 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:29:59,242 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:29:59,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:00,944 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:01,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:01,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:03,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:03,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:03,805 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:04,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:30:04,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:30:04,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:30:06,288 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:06,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:06,341 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:08,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:08,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:09,807 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:11,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:11,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:11,742 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:13,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:13,348 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:15,809 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:16,141 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:16,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:17,678 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:18,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:18,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:19,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:30:19,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:30:19,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:30:19,353 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:30:19,810 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:20,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:20,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:23,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:23,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:23,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:25,813 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:27,160 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:30:27,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:27,193 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:28,438 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:29,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:29,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:31,815 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:32,009 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:32,024 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:34,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:34,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:34,128 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:34,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:30:34,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:30:34,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:30:35,816 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:37,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:37,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:39,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:39,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:39,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:41,818 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:42,427 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:42,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:44,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:44,488 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:45,057 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:46,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:46,526 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:47,820 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:48,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:48,566 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:49,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:30:49,161 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:30:49,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:30:50,817 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:30:50,818 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:51,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:51,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:53,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:53,822 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:30:53,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:55,916 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:30:56,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:56,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:58,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:58,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:30:59,824 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:01,711 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:01,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:01,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:03,825 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:04,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:31:04,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:31:04,181 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:31:05,554 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:31:05,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:05,597 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:07,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:07,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:07,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:09,827 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:10,637 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:10,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:12,643 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:12,699 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:12,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:14,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:14,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:15,829 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:16,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:16,829 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:18,261 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:19,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:31:19,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:31:19,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:31:19,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:19,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:19,831 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:20,385 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:31:21,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:21,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:23,386 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:24,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:24,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:25,833 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:26,484 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:26,501 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:28,460 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:29,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:29,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:31,746 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:31,768 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:31,834 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:33,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:33,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:34,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:31:34,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:31:34,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:31:34,323 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:35,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:35,882 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:37,836 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:39,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:39,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:39,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:42,686 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:31:42,708 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:42,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:43,838 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:44,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:44,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:45,074 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:47,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:47,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:47,840 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:49,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:31:49,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:31:49,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:31:49,357 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:31:49,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:49,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:50,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:52,215 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:31:52,216 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:31:52,216 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:31:52,217 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:31:52,842 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:31:52,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:52,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:53,842 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:31:54,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:54,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:56,218 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:31:56,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:56,995 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:59,017 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:59,038 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:31:59,844 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:01,868 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:02,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:02,212 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:04,161 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:32:04,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:32:04,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:32:04,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:04,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:05,846 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:06,827 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:07,294 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:07,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:09,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:09,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:11,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:11,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:11,849 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:12,778 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:14,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:14,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:15,850 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:16,180 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:16,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:18,600 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:19,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:32:19,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:32:19,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:32:19,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:32:20,124 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:32:20,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:20,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:21,852 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:22,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:22,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:24,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:25,365 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:25,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:25,853 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:27,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:27,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:29,650 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:30,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:30,478 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:31,856 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:32,509 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:32,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:34,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:32:34,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:32:34,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:32:35,386 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:35,546 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:35,563 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:37,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:37,614 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:37,858 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:40,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:40,715 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:40,993 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:42,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:42,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:43,860 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:44,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:44,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:45,993 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:47,982 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:48,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:49,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:32:49,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:32:49,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:32:49,361 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:32:49,862 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:50,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:50,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:51,362 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:52,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:52,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:53,863 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:32:55,100 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:55,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:56,402 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:32:58,793 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:32:58,813 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:58,823 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:32:59,865 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:01,734 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:01,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:01,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:03,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:03,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:33:04,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:33:04,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:33:05,866 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:06,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:06,651 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:07,381 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:08,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:08,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:09,869 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:10,706 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:10,717 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:12,483 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:13,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:13,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:15,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:15,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:15,870 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:18,246 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:18,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:18,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:19,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:33:19,162 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:33:19,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:33:19,387 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:33:20,276 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:20,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:21,872 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:23,226 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:23,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:24,153 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:25,261 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:25,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:27,874 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:28,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:28,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:29,282 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:30,118 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:30,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:33,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:33,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:33,877 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:34,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:33:34,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:33:34,182 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:33:34,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:36,644 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:33:36,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:36,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:37,878 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:38,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:38,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:40,154 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:41,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:41,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:43,414 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:43,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:43,881 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:45,978 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:46,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:46,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:48,267 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:48,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:49,162 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:33:49,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:33:49,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:33:49,364 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:33:49,883 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:51,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:51,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:51,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:53,312 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:53,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:53,885 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:33:56,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:56,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:56,726 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:33:58,271 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:58,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:33:59,886 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:01,109 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:01,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:02,642 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:03,138 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:03,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:34:04,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:34:04,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:34:05,889 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:05,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:06,008 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:08,030 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:08,040 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:08,335 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:09,386 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:34:09,388 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:34:09,388 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:34:09,389 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:34:09,890 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:34:09,891 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:10,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:10,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:11,891 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:13,389 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:14,714 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:34:14,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:14,749 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:17,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:17,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:17,893 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:34:19,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:34:19,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:34:19,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:19,680 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:20,963 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:34:20,964 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:22,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:22,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:23,895 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:24,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:24,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:25,965 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:27,631 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:27,644 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:27,896 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:29,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:29,674 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:31,124 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:32,502 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:32,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:33,898 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:34,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:34:34,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:34:34,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:34:34,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:34,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:36,353 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:37,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:37,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:37,899 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:39,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:39,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:41,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:41,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:41,810 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:43,901 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:44,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:44,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:46,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:46,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:47,549 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:49,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:34:49,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:34:49,183 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:34:49,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:49,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:49,903 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:50,835 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:34:52,836 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:52,927 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:34:52,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:52,961 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:55,905 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:34:55,993 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:56,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:58,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:58,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:34:58,453 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:34:59,906 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:00,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:00,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:02,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:02,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:03,734 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:35:04,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:35:04,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:35:05,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:05,887 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:05,909 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:07,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:07,915 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:09,370 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:10,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:10,734 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:11,912 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:12,752 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:12,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:14,984 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:15,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:15,636 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:15,914 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:17,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:17,665 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:35:19,163 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:35:19,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:35:20,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:20,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:20,942 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:35:20,943 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:21,916 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:22,514 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:22,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:25,330 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:25,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:25,945 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:27,373 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:27,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:27,917 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:30,750 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:35:30,775 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:30,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:31,318 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:33,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:33,496 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:33,919 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:34,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:35:34,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:35:34,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:35:35,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:35,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:36,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:38,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:38,586 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:39,921 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:40,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:40,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:41,407 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:42,641 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:42,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:43,922 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:45,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:45,591 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:47,289 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:47,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:47,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:49,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:35:49,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:35:49,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:35:49,925 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:50,432 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:35:50,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:50,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:52,433 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:52,573 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:52,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:55,409 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:55,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:55,927 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:35:57,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:57,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:35:57,845 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:35:59,929 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:00,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:00,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:02,445 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:02,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:03,681 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:04,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:36:04,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:36:04,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:36:05,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:05,282 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:05,931 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:08,841 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:36:08,863 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:08,878 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:09,520 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:11,911 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:11,920 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:11,933 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:13,943 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:13,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:14,521 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:15,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:15,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:17,936 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:18,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:18,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:19,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:36:19,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:36:19,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:36:19,372 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:36:20,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:20,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:20,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:22,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:22,850 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:23,943 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:25,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:25,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:25,910 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:25,981 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:36:25,983 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:36:25,983 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:36:25,983 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:36:26,947 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:36:27,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:27,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:27,947 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:30,423 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:30,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:30,984 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:32,455 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:32,466 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:33,949 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:34,163 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:36:34,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:36:34,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:36:35,623 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:35,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:36,380 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:37,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:37,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:37,950 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:40,803 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:40,814 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:41,996 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:42,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:42,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:43,953 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:46,224 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:36:46,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:46,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:47,526 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:49,083 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:49,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:49,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:36:49,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:36:49,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:36:49,955 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:50,292 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:36:51,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:51,140 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:53,000 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:53,957 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:36:53,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:53,989 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:56,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:56,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:58,001 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:36:58,050 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:58,061 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:36:59,959 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:00,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:00,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:02,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:02,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:03,849 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:04,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:37:04,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:37:04,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:37:04,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:04,896 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:05,961 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:07,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:07,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:09,541 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:09,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:09,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:11,963 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:12,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:12,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:14,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:14,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:14,626 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:17,275 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:17,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:17,964 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:19,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:37:19,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:37:19,184 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:37:19,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:19,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:19,375 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:37:20,201 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:21,966 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:22,708 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:37:22,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:22,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:25,202 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:25,645 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:25,655 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:27,675 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:27,687 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:27,967 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:30,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:30,548 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:31,046 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:32,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:32,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:33,969 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:34,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:37:34,164 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:37:34,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:37:35,410 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:35,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:36,291 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:37,460 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:37,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:37,971 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:40,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:40,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:41,722 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:42,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:42,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:43,972 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:45,044 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:45,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:47,094 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:47,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:47,611 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:49,164 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:37:49,165 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:37:49,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:37:49,377 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:37:49,941 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:49,955 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:49,974 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:51,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:52,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:53,378 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:37:55,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:55,056 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:55,976 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:37:57,077 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:57,092 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:37:58,449 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:00,395 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:38:00,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:00,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:01,978 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:03,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:03,194 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:04,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:38:04,165 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:38:04,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:38:04,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:05,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:05,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:05,981 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:08,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:08,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:09,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:10,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:10,237 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:11,982 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:13,188 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:13,210 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:15,073 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:15,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:15,242 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:17,984 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:18,065 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:18,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:38:19,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:38:19,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:38:19,379 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:38:20,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:20,146 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:20,380 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:21,986 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:23,042 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:23,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:25,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:25,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:25,671 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:27,871 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:27,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:27,988 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:29,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:29,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:31,268 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:32,729 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:32,748 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:33,990 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:34,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:38:34,165 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:38:34,185 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:38:34,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:34,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:36,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:37,992 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:38,105 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:38:38,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:38,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:40,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:40,936 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:42,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:42,868 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:38:42,869 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:38:42,870 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:38:42,870 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:38:42,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:42,971 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:42,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:38:43,994 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:44,995 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:45,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:45,806 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:47,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:47,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:47,873 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:38:49,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:38:49,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:38:49,394 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:38:50,584 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:50,601 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:52,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:52,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:53,396 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:38:55,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:55,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:55,998 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:38:57,457 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:57,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:38:59,365 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:00,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:00,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:02,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:02,236 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:04,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:39:04,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:39:04,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:39:05,075 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:05,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:05,343 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:07,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:07,149 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:09,879 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:09,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:10,003 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:39:10,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:11,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:11,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:15,285 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:39:15,305 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:15,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:15,532 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:17,935 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:17,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:19,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:39:19,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:39:19,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:39:19,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:19,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:20,908 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:39:20,908 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:22,008 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:39:22,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:22,782 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:24,799 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:24,809 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:25,910 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:27,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:27,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:29,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:29,642 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:30,910 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:32,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:32,476 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:39:34,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:39:34,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:39:34,498 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:34,510 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:36,013 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:39:36,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:37,219 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:37,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:39,272 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:39,283 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:41,366 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:42,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:42,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:44,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:44,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:46,721 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:46,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:46,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:48,017 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:39:48,924 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:48,938 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:49,165 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:39:49,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:39:49,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:39:49,384 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:39:52,385 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:39:52,439 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:39:52,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:52,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:55,258 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:55,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:57,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:57,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:39:57,386 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:00,021 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:00,172 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:00,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:02,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:02,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:03,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:40:04,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:40:04,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:40:05,190 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:05,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:07,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:07,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:08,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:10,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:10,088 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:12,025 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:12,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:12,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:14,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:14,963 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:14,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:17,010 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:17,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:19,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:40:19,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:40:19,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:40:19,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:19,386 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:40:19,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:19,958 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:22,027 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:22,046 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:24,029 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:24,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:24,285 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:24,757 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:26,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:26,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:29,758 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:29,803 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:40:29,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:29,842 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:32,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:32,561 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:40:34,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:40:34,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:40:34,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:34,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:35,328 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:37,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:37,638 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:38,033 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:39,656 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:39,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:40,817 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:42,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:42,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:44,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:44,469 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:45,818 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:47,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:47,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:49,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:40:49,166 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:40:49,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:40:49,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:49,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:50,037 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:40:51,134 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:40:51,135 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:52,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:52,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:54,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:54,239 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:56,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:40:57,087 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:57,108 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:59,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:40:59,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:01,761 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:01,909 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:01,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:02,041 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:03,970 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:03,985 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:04,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:41:04,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:41:04,186 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:41:07,277 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:41:07,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:07,311 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:07,376 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:10,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:10,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:12,116 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:12,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:12,377 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:14,045 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:14,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:14,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:16,922 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:16,931 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:18,066 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:19,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:41:19,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:41:19,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:41:19,821 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:19,838 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:20,174 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:41:21,855 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:21,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:23,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:24,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:24,759 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:26,049 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:26,808 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:26,836 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:28,222 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:29,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:29,545 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:31,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:31,582 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:33,223 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:34,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:41:34,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:41:34,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:41:34,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:34,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:36,450 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:36,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:38,053 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:38,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:39,362 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:39,385 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:41,424 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:41,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:43,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:44,626 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:41:44,647 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:44,663 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:47,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:47,538 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:48,355 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:49,166 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:41:49,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:41:49,187 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:41:49,391 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:41:49,558 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:49,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:52,058 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:41:52,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:52,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:53,530 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:54,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:54,345 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:57,232 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:57,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:58,531 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:41:59,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:41:59,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:02,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:02,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:04,063 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:42:04,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:42:04,185 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:04,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:42:04,207 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:04,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:07,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:07,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:09,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:09,076 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:09,358 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:11,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:11,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:13,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:13,937 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:14,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:16,066 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:16,703 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:16,724 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:18,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:18,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:42:19,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:42:19,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:42:19,392 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:42:19,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:22,098 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:42:22,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:22,128 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:24,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:24,764 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:24,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:26,796 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:26,807 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:28,069 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:29,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:29,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:29,954 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:31,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:31,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:42:34,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:42:34,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:42:34,567 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:34,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:35,344 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:36,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:36,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:39,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:39,435 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:40,072 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:41,308 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:41,452 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:41,465 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:44,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:44,324 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:46,309 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:46,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:46,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:49,104 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:49,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:49,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:42:49,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:42:49,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:42:50,578 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:42:51,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:51,155 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:51,579 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:54,076 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:42:54,097 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:54,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:56,133 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:56,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:56,949 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:42:59,480 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:42:59,500 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:42:59,511 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:01,949 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:02,336 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:02,354 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:43:04,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:43:04,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:43:04,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:04,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:06,080 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:07,248 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:07,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:07,470 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:09,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:09,301 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:12,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:12,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:12,470 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:14,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:14,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:17,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:17,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:18,083 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:18,094 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:19,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:19,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:19,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:43:19,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:43:19,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:43:19,396 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:43:21,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:21,976 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:23,397 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:23,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:24,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:26,772 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:26,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:28,398 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:28,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:28,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:31,787 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:31,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:32,087 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:33,629 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:33,822 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:33,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:34,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:43:34,167 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:43:34,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:43:37,360 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:43:37,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:37,395 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:39,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:40,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:40,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:42,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:42,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:44,091 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:44,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:44,870 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:44,881 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:46,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:46,897 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:49,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:43:49,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:43:49,188 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:43:49,397 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:43:49,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:49,741 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:50,399 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:51,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:51,771 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:54,543 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:54,553 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:56,095 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:43:56,210 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:43:56,572 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:56,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:59,398 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:43:59,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:01,210 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:01,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:01,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:04,167 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:44:04,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:44:04,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:44:04,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:04,213 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:06,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:06,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:06,368 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:08,099 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:09,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:09,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:11,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:11,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:11,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:14,687 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:44:14,713 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:14,728 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:16,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:17,349 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:17,363 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:19,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:44:19,168 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:44:19,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:44:19,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:19,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:20,103 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:21,409 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:44:21,410 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:22,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:22,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:24,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:24,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:26,411 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:27,429 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:27,442 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:29,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:29,474 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:32,077 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:32,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:32,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:33,107 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:34,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:44:34,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:44:34,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:44:34,320 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:34,328 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:37,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:37,081 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:37,337 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:39,101 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:39,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:42,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:42,111 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:42,338 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:44,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:44,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:45,110 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:47,068 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:47,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:47,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:49,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:49,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:44:49,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:44:49,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:44:49,402 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:44:52,353 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:44:52,376 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:52,386 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:53,403 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:55,107 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:55,117 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:57,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:57,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:58,738 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:44:59,115 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:44:59,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:44:59,886 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:01,904 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:01,916 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:03,739 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:04,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:45:04,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:45:04,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:45:04,750 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:04,760 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:06,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:06,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:09,273 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:09,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:09,587 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:11,119 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:45:11,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:11,615 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:14,273 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:14,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:14,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:16,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:16,473 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:19,168 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:45:19,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:45:19,189 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:45:19,197 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:19,209 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:19,306 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:19,403 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:45:21,228 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:21,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:23,123 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:45:23,980 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:23,991 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:24,490 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:26,012 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:26,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:29,388 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:45:29,411 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:29,421 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:29,490 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:32,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:32,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:34,153 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:34,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:45:34,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:45:34,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:45:35,127 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:45:35,354 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:37,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:37,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:39,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:39,058 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:40,356 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:41,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:41,893 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:43,913 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:43,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:45,790 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:45,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:45,959 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:47,131 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:45:48,579 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:48,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:45:49,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:45:49,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:45:49,404 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:45:50,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:50,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:51,405 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:53,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:53,353 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:55,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:55,407 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:56,406 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:45:58,231 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:58,250 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:45:59,136 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:00,270 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:00,293 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:01,702 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:03,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:03,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:46:04,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:46:04,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:46:06,739 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:46:06,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:06,770 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:07,363 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:09,697 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:09,714 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:11,139 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:11,733 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:11,743 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:12,482 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:14,479 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:14,489 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:16,507 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:16,518 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:17,482 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:46:19,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:46:19,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:46:19,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:19,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:21,270 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:46:21,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:21,428 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:22,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:23,142 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:24,420 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:24,433 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:26,451 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:26,461 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:27,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:29,313 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:29,323 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:31,342 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:31,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:32,605 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:46:34,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:46:34,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:46:34,279 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:34,292 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:36,317 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:36,335 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:37,147 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:38,311 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:39,254 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:39,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:41,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:41,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:43,312 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:44,567 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:46:44,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:44,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:47,396 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:47,406 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:48,596 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:49,150 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:46:49,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:46:49,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:46:49,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:46:49,407 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:46:49,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:49,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:52,256 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:52,268 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:54,286 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:54,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:54,409 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:46:57,184 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:57,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:59,216 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:46:59,224 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:00,093 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:01,154 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:01,977 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:01,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:04,014 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:04,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:04,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:47:04,169 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:47:04,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:47:05,324 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:06,844 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:06,857 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:08,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:08,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:10,325 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:11,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:11,684 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:13,158 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:13,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:13,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:15,904 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:16,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:16,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:18,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:18,468 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:19,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:47:19,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:47:19,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:47:19,409 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:47:21,081 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:21,842 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:47:21,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:21,877 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:23,161 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:24,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:24,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:26,688 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:27,162 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:27,776 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:27,794 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:29,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:30,942 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:32,277 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:32,972 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:33,164 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:34,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:34,169 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:47:34,480 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:47:34,481 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:47:37,503 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:37,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:37,867 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:39,166 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:39,542 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:39,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:42,454 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:42,485 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:42,868 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:44,525 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:44,536 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:47,296 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:47,316 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:47,869 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:47:49,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:47:49,190 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:47:49,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:49,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:49,410 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:47:51,170 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:47:52,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:52,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:53,790 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:54,186 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:54,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:56,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:56,960 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:58,790 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:47:58,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:47:59,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:01,174 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:02,294 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:48:02,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:02,326 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:48:04,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:48:04,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:48:04,375 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:05,013 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:05,023 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:07,048 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:07,059 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:09,375 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:10,022 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:10,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:12,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:12,064 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:13,178 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:14,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:14,927 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:15,035 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:16,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:16,967 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:19,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:48:19,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:48:19,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:48:19,411 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:48:19,835 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:19,852 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:20,964 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:21,181 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:21,881 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:21,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:24,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:24,805 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:25,965 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:26,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:26,848 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:29,183 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:29,769 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:29,786 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:31,684 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:31,830 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:31,843 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:48:34,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:48:34,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:48:34,589 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:34,608 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:36,654 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:36,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:37,413 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:39,910 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:48:39,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:39,944 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:42,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:42,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:43,187 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:43,373 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:44,811 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:44,833 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:47,626 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:47,640 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:48,374 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:48:49,170 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:48:49,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:48:49,413 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:48:49,662 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:49,671 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:52,532 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:52,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:53,191 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:48:54,078 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:54,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:54,605 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:57,530 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:57,552 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:59,080 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:48:59,574 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:48:59,593 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:02,379 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:02,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:49:04,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:49:04,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:49:04,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:04,440 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:04,449 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:05,195 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:07,295 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:07,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:09,294 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:09,344 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:09,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:12,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:12,121 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:14,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:14,160 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:14,937 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:15,199 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:17,474 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:49:17,497 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:17,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:19,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:49:19,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:49:19,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:49:20,159 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:20,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:20,540 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:49:20,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:22,187 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:22,196 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:25,026 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:25,035 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:25,542 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:27,055 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:27,066 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:29,203 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:30,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:30,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:30,990 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:32,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:32,119 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:34,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:49:34,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:49:34,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:49:34,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:34,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:36,907 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:36,917 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:36,959 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:37,206 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:39,686 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:39,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:41,745 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:41,754 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:41,960 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:44,508 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:44,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:45,209 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:46,550 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:46,576 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:47,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:49,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:49:49,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:49:49,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:49:49,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:49,360 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:49,416 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:49:51,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:51,399 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:52,417 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:54,705 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:49:54,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:54,735 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:57,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:57,583 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:58,126 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:49:59,214 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:49:59,602 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:49:59,612 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:02,416 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:02,426 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:03,127 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:04,170 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:50:04,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:50:04,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:50:04,443 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:04,453 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:07,273 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:07,300 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:08,257 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:09,217 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:09,318 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:09,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:12,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:12,122 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:13,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:14,158 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:14,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:16,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:16,951 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:18,259 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:18,974 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:18,987 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:50:19,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:50:19,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:50:19,417 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:50:21,221 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:21,949 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:21,966 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:23,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:23,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:24,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:26,930 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:26,950 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:28,554 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:28,990 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:29,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:31,225 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:32,269 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:50:32,291 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:32,303 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:33,866 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:34,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:50:34,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:50:34,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:50:35,006 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:35,016 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:37,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:37,047 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:39,359 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:39,849 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:39,861 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:41,880 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:41,895 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:43,229 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:44,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:44,883 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:45,113 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:46,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:46,929 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:49,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:50:49,171 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:50:49,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:50:49,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:49,744 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:50,665 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:50:50,667 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:51,778 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:51,789 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:54,551 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:54,575 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:55,666 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:50:56,599 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:56,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:57,234 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:50:59,397 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:50:59,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:00,758 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:01,434 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:01,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:04,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:51:04,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:51:04,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:04,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:51:04,201 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:05,238 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:06,240 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:06,249 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:06,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:09,493 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:51:09,512 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:09,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:11,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:12,215 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:12,225 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:14,243 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:14,253 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:16,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:17,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:17,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:19,132 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:19,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:19,171 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:51:19,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:51:19,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:51:19,242 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:20,638 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:51:21,639 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:22,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:22,134 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:24,170 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:24,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:26,948 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:26,962 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:27,251 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:27,599 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:28,986 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:29,000 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:31,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:31,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:32,599 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:33,812 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:33,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:51:34,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:51:34,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:51:36,610 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:36,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:38,414 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:38,652 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:38,668 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:41,257 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:41,617 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:41,639 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:43,454 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:43,683 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:43,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:46,943 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:51:46,964 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:46,978 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:48,455 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:49,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:51:49,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:51:49,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:51:49,260 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:51:49,423 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:51:49,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:49,667 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:51,691 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:51,700 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:54,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:54,529 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:54,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:56,559 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:56,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:59,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:51:59,519 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:51:59,531 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:01,569 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:01,580 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:03,265 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:04,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:52:04,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:52:04,192 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:52:04,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:04,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:05,335 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:06,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:06,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:09,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:09,199 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:10,336 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:11,220 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:11,241 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:11,268 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:14,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:14,034 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:15,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:16,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:16,078 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:18,847 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:18,868 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:19,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:52:19,172 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:52:19,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:52:19,424 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:52:20,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:20,901 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:21,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:24,225 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:52:24,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:24,255 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:25,277 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:27,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:27,063 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:27,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:29,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:29,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:31,894 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:31,905 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:32,390 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:33,280 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:33,923 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:33,933 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:52:34,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:52:34,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:52:36,761 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:36,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:38,347 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:38,817 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:38,832 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:41,588 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:41,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:43,348 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:43,632 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:43,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:46,578 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:46,600 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:47,286 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:48,618 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:48,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:48,958 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:49,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:52:49,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:52:49,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:52:50,828 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:52:51,408 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:51,438 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:53,464 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:53,480 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:54,550 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:52:55,288 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:52:56,238 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:56,259 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:58,302 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:58,315 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:52:59,553 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:01,579 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:53:01,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:01,613 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:04,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:53:04,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:53:04,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:53:04,309 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:04,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:05,293 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:05,333 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:06,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:06,352 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:09,139 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:09,150 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:10,334 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:11,169 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:11,179 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:14,135 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:14,152 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:15,296 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:15,745 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:16,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:16,204 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:18,973 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:18,992 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:19,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:53:19,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:53:19,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:53:19,428 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:53:21,036 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:21,045 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:21,429 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:23,802 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:23,826 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:25,845 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:25,862 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:26,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:28,658 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:28,672 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:29,302 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:30,710 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:30,719 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:31,644 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:33,470 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:33,483 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:34,172 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:53:34,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:53:34,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:53:35,524 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:35,533 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:37,384 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:38,822 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:53:38,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:38,864 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:39,306 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:41,695 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:41,705 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:42,734 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:43,726 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:43,738 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:46,556 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:46,568 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:47,735 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:48,592 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:48,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:53:49,173 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:53:49,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:53:49,430 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:53:51,577 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:51,596 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:53,004 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:53,311 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:53:53,619 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:53,635 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:56,432 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:56,447 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:58,005 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:53:58,486 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:53:58,499 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:01,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:01,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:03,006 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:03,310 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:03,325 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:04,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:54:04,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:54:04,193 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:54:05,317 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:06,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:06,125 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:08,166 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:08,175 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:08,628 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:11,080 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:11,100 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:13,120 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:13,136 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:13,629 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:16,410 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:54:16,430 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:16,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:17,321 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:19,103 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:19,115 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:19,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:54:19,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:54:19,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:54:19,392 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:19,433 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:54:21,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:21,154 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:23,322 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:23,947 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:23,957 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:25,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:25,979 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:25,988 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:28,831 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:28,851 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:30,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:30,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:31,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:31,330 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:33,646 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:33,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:34,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:54:34,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:54:34,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:54:35,702 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:35,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:36,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:38,629 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:38,653 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:40,673 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:40,685 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:41,321 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:43,334 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:43,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:43,527 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:45,554 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:45,571 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:46,897 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:48,346 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:48,366 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:49,173 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:54:49,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:54:49,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:54:49,433 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:54:50,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:50,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:51,979 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:53,337 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:54:53,689 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:54:53,711 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:53,722 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:56,516 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:56,537 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:56,986 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:54:58,555 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:54:58,564 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:01,361 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:01,372 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:01,987 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:03,393 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:03,403 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:04,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:55:04,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:55:04,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:55:06,265 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:06,284 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:07,341 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:07,822 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:08,307 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:08,321 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:11,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:11,091 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:12,822 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:13,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:13,143 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:16,037 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:16,052 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:17,823 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:18,093 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:18,102 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:55:19,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:55:19,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:55:19,344 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:19,435 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:55:20,898 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:20,919 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:22,945 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:22,956 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:23,436 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:25,721 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:25,736 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:27,785 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:27,797 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:29,224 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:29,347 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:31,041 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:55:31,062 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:31,072 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:33,747 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:33,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:55:34,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:55:34,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:55:34,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:35,781 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:35,792 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:38,594 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:38,604 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:39,320 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:40,622 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:40,633 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:41,351 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:43,603 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:43,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:44,561 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:45,650 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:45,664 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:48,400 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:48,418 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:49,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:55:49,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:55:49,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:55:49,437 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:55:50,438 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:50,441 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:50,459 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:51,354 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:55:53,314 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:53,334 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:55,357 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:55,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:55,694 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:55:58,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:55:58,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:00,347 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:00,359 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:00,695 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:03,124 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:03,145 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:04,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:56:04,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:56:04,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:56:05,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:05,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:05,359 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:06,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:08,771 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:56:08,788 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:08,798 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:10,816 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:10,825 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:11,430 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:13,657 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:13,670 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:15,689 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:15,704 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:17,136 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:17,362 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:17,730 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:17,753 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:56:19,174 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:56:19,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:56:20,628 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:20,648 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:20,881 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:56:22,692 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:22,701 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:22,882 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:25,565 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:25,581 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:27,620 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:27,630 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:27,884 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:29,365 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:30,431 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:30,448 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:32,493 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:32,504 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:33,796 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:34,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:56:34,175 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:56:34,194 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:56:35,269 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:35,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:37,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:37,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:38,966 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:40,067 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:40,089 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:40,368 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:42,114 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:42,127 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:44,010 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:45,392 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:56:45,412 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:45,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:48,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:48,144 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:49,011 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:49,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:56:49,175 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:56:49,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:56:49,440 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:56:50,164 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:50,173 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:53,129 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:53,142 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:54,373 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:56:54,562 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:56:55,162 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:55,176 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:57,997 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:58,018 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:56:59,563 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:00,043 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:00,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:02,839 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:02,869 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:04,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:57:04,176 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:57:04,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:57:04,376 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:04,890 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:04,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:05,391 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:07,834 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:07,854 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:09,872 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:09,885 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:10,393 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:12,661 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:12,676 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:14,716 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:14,725 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:15,394 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:17,521 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:17,539 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:18,380 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:19,174 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:57:19,175 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:57:19,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:57:19,557 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:19,570 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:20,600 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:57:20,601 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:23,072 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:57:23,095 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:23,106 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:25,601 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:26,371 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:26,382 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:28,383 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:28,402 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:28,415 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:30,759 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:31,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:31,339 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:33,356 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:33,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:34,175 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:57:34,175 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:57:34,195 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:57:36,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:36,192 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:36,254 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: partial_history +2024-03-27 20:57:36,256 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,261 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: metric +2024-03-27 20:57:36,438 DEBUG SenderThread:1893870 [sender.py:send():382] send: history +2024-03-27 20:57:36,439 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: summary_record +2024-03-27 20:57:36,439 INFO SenderThread:1893870 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-27 20:57:37,385 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/wandb-summary.json +2024-03-27 20:57:37,386 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/config.yaml +2024-03-27 20:57:38,211 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:38,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:38,386 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:41,015 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:41,025 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:41,440 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:43,041 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:43,049 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:46,021 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:46,039 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:46,389 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:57:47,191 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:48,060 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:48,070 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:49,443 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:57:49,454 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:57:49,454 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:57:49,501 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:57:50,758 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:50,779 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:52,646 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:52,801 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:53,859 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:55,888 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:55,903 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:57,647 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:57:58,666 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:57:58,846 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:02,447 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:58:02,475 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:02,487 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:02,648 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:04,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:58:04,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:58:04,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:58:05,002 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:05,222 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:07,825 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:08,252 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:08,263 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:08,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/config.yaml +2024-03-27 20:58:08,395 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:10,287 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:10,308 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:12,329 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:12,396 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:13,284 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:13,405 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:15,425 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:15,439 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:16,397 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:18,340 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:18,342 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:18,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:19,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:58:19,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:58:19,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:58:19,573 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:58:20,370 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:20,380 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:22,399 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:23,367 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:23,384 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:23,394 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:25,413 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:25,422 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:28,289 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:28,304 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:28,401 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:28,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:30,322 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:30,332 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:33,350 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:33,369 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:33,404 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:34,403 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:34,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:58:34,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:58:34,427 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:58:35,390 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:35,401 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:38,367 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:38,378 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:38,556 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:40,405 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:41,976 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:58:41,994 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:42,005 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:44,465 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:45,054 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:45,079 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:46,406 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:47,099 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:47,112 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:49,425 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:58:49,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:58:49,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:58:49,551 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:58:49,552 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:50,098 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:50,110 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:52,131 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:52,148 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:52,409 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:58:55,165 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:55,177 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:55,522 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:58:57,200 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:57,214 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:58:58,413 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:59:00,171 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:00,182 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:00,540 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:02,202 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:02,217 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:04,415 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:59:04,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:59:04,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:59:04,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:59:05,183 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:05,195 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:05,633 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:07,221 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:07,230 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:10,233 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:10,245 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:10,416 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:59:11,604 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:12,264 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:12,274 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:15,288 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:15,299 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:16,418 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log +2024-03-27 20:59:16,640 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:17,319 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:17,331 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:19,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-27 20:59:19,426 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: stop_status +2024-03-27 20:59:19,426 DEBUG SenderThread:1893870 [sender.py:send_request():409] send_request: stop_status +2024-03-27 20:59:20,576 DEBUG SenderThread:1893870 [sender.py:send():382] send: stats +2024-03-27 20:59:20,834 ERROR gpu :1893870 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-03-27 20:59:20,860 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:20,876 ERROR gpu :1893870 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-03-27 20:59:21,668 DEBUG HandlerThread:1893870 [handler.py:handle_request():146] handle_request: status_report +2024-03-27 20:59:22,420 INFO Thread-13 :1893870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/distil-large-v3-hi-ft/wandb/run-20240327_190418-lwtbcr8s/files/output.log