diff --git "a/wandb/run-20220120_135649-r5vgtpr6/logs/debug-internal.log" "b/wandb/run-20220120_135649-r5vgtpr6/logs/debug-internal.log" --- "a/wandb/run-20220120_135649-r5vgtpr6/logs/debug-internal.log" +++ "b/wandb/run-20220120_135649-r5vgtpr6/logs/debug-internal.log" @@ -2322,3 +2322,766 @@ 2022-01-20 14:56:14,433 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status 2022-01-20 14:56:14,433 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status 2022-01-20 14:56:27,163 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 14:56:30,010 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:56:30,011 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:56:32,139 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:34,140 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:36,141 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:38,141 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:40,142 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:42,143 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:44,144 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:45,144 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:56:45,145 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:56:46,146 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:48,146 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:50,147 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:52,148 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:54,149 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:56,150 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:56:57,236 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 14:56:58,151 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:00,152 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:00,362 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:57:00,362 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:57:02,153 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:04,153 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:06,154 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:08,155 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:10,156 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:12,157 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:14,158 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:15,501 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:57:15,502 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:57:16,158 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:18,159 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:20,161 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:22,162 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:24,163 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:26,163 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:27,309 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 14:57:28,164 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:30,165 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:30,645 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:57:30,645 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:57:32,167 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:34,168 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:36,169 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:38,170 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:40,171 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:42,172 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:44,173 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:45,782 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:57:45,782 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:57:46,174 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:48,175 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:50,176 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:52,177 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:54,178 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:56,179 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:57:57,383 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 14:57:58,180 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:00,181 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:00,947 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:58:00,948 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:58:02,182 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:04,183 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:06,184 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:08,185 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:10,186 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:12,187 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:14,188 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:16,091 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:58:16,092 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:58:16,189 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:18,190 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:20,191 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:22,191 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:24,192 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:26,193 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:27,455 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 14:58:28,194 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:30,195 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:31,240 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:58:31,241 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:58:32,198 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:35,200 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:37,200 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:39,201 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:41,202 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:43,203 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:45,204 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:46,521 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:58:46,521 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:58:47,205 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:49,206 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:51,207 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:53,208 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:55,209 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:57,210 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:58:57,530 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 14:58:59,211 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:01,212 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:01,664 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:59:01,664 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:59:03,213 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:05,214 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:07,215 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:09,215 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:11,216 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:13,217 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:15,218 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:16,800 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:59:16,800 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:59:17,220 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:19,221 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:21,221 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:23,222 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:25,223 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:27,224 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:27,604 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 14:59:29,225 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:31,225 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:31,940 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:59:31,941 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:59:33,226 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:35,227 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:37,228 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:39,229 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:41,230 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:43,231 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:45,232 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:47,087 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 14:59:47,087 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 14:59:47,233 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:49,234 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:51,235 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:53,236 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:55,237 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:57,238 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 14:59:57,679 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 14:59:59,239 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:01,240 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:02,226 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:00:02,226 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:00:03,241 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:05,242 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:07,243 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:09,244 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:11,245 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:13,246 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:15,247 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:17,247 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:17,365 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:00:17,365 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:00:19,248 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:21,249 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:23,251 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:25,252 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:27,253 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:27,751 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:00:29,254 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:31,255 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:32,504 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:00:32,504 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:00:33,256 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:35,257 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:37,258 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:39,258 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:41,259 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:43,260 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:45,261 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:47,262 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:47,639 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:00:47,639 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:00:49,263 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:51,264 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:53,265 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:55,266 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:57,267 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:00:57,821 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:00:59,268 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:01,269 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:02,777 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:01:02,777 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:01:03,270 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:05,271 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:07,272 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:09,273 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:11,274 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:13,275 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:15,276 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:17,277 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:17,911 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:01:17,911 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:01:19,278 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:21,278 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:23,279 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:25,280 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:27,281 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:27,890 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:01:29,282 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:31,283 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:33,048 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:01:33,049 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:01:33,284 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:35,285 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:37,286 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:39,287 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:41,288 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:43,289 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:45,290 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:47,291 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:48,189 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:01:48,190 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:01:49,292 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:51,293 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:53,294 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:55,295 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:57,296 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:01:57,962 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:01:59,297 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:01,297 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:03,298 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:03,331 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:02:03,331 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:02:05,299 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:07,300 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:09,301 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:11,302 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:13,303 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:15,304 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:17,304 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:18,497 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:02:18,498 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:02:19,305 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:21,306 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:23,307 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:25,308 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:27,309 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:28,036 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:02:29,310 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:31,311 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:33,312 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:33,637 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:02:33,637 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:02:35,313 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:37,314 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:39,315 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:41,316 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:43,317 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:45,317 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:47,318 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:48,772 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:02:48,773 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:02:49,319 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:51,320 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:53,321 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:55,322 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:57,322 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:02:58,112 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:02:59,323 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:01,324 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:03,325 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:03,908 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:03:03,909 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:03:05,326 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:07,327 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:09,327 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:11,328 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:13,329 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:15,330 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:17,331 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:19,257 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:03:19,258 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:03:19,333 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:21,334 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:23,334 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:25,335 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:27,336 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:28,188 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:03:29,337 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:31,338 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:33,339 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:34,408 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:03:34,408 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:03:35,340 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:37,341 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:39,342 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:41,343 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:43,344 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:45,345 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:47,346 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:49,347 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:49,548 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:03:49,549 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:03:51,348 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:53,349 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:55,350 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:57,351 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:03:58,262 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:03:59,352 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:01,353 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:03,354 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:04,686 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:04:04,686 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:04:05,355 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:07,356 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:09,357 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:11,358 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:13,359 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:15,360 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:17,361 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:19,362 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:19,826 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:04:19,826 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:04:21,363 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:23,364 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:25,365 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:27,366 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:28,330 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:04:29,367 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:31,368 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:33,369 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:34,962 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:04:34,962 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:04:35,370 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:37,371 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:39,372 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:41,373 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:43,374 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:45,374 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:47,375 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:49,377 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:50,096 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:04:50,097 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:04:51,378 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:53,378 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:55,379 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:57,380 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:04:58,400 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:04:59,381 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:01,382 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:03,383 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:05,240 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:05:05,240 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:05:05,384 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:07,385 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:09,386 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:11,387 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:13,388 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:15,389 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:17,390 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:19,391 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:20,378 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:05:20,379 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:05:21,392 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:23,393 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:25,394 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:27,395 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:28,471 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:05:29,396 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:31,397 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:33,398 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:35,399 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:35,510 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:05:35,510 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:05:37,400 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:39,400 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:41,401 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:43,402 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:45,403 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:47,404 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:49,405 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:50,660 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:05:50,661 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:05:51,406 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:53,407 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:55,408 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:57,409 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:05:58,543 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:05:59,410 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:01,410 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:03,411 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:05,412 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:05,799 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:06:05,799 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:06:07,413 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:09,414 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:11,415 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:13,416 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:15,417 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:17,418 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:19,419 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:20,940 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:06:20,941 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:06:21,420 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:23,421 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:25,422 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:27,423 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:28,607 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:06:29,424 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:31,425 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:33,426 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:35,427 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:36,087 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:06:36,087 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:06:37,428 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:39,429 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:41,429 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:43,431 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:45,432 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:47,433 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:50,434 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:51,301 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:06:51,302 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:06:52,435 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:54,437 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:56,438 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:58,439 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:06:58,671 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:07:00,440 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:02,441 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:04,442 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:06,442 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:07:06,443 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:07:06,444 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:08,444 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:10,445 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:12,447 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:14,448 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:16,449 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:18,450 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:20,451 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:21,584 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:07:21,584 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:07:22,452 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:24,453 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:26,454 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:28,455 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:28,736 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:07:30,456 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:32,457 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:34,458 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:36,459 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:36,778 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:07:36,779 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:07:38,460 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:40,461 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:42,462 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:44,463 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:46,464 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:48,465 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:50,466 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:51,916 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:07:51,916 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:07:52,467 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:54,469 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:56,470 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:58,471 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:07:58,803 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:08:00,472 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:02,473 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:04,474 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:06,475 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:07,056 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:08:07,057 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:08:08,476 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:10,477 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:12,477 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:14,478 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:16,479 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:18,480 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:20,481 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:22,206 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:08:22,207 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:08:22,482 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:24,483 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:26,484 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:28,485 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:28,868 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:08:30,486 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:32,487 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:34,488 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:36,489 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:37,346 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:08:37,346 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:08:38,490 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:40,491 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:42,492 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:44,493 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:46,494 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:48,495 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:50,496 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:52,487 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:08:52,487 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:08:52,497 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:54,498 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:56,498 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:58,499 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:08:58,935 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:09:00,500 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:02,501 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:04,502 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:06,503 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:07,627 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:09:07,627 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:09:08,504 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:10,505 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:12,506 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:14,507 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:16,508 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:18,509 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:20,510 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:22,511 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:23,088 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:09:23,088 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:09:24,512 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:26,513 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:28,514 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:29,008 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:09:30,515 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:32,516 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:34,517 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:36,518 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:38,235 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:09:38,236 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:09:38,519 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:40,520 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:42,521 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:44,522 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:46,523 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:48,524 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:50,525 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:52,526 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:53,400 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:09:53,400 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:09:54,527 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:56,528 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:58,529 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:09:59,085 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:10:00,530 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:02,531 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:04,532 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:06,533 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:08,534 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:08,536 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:10:08,537 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:10:10,535 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:12,536 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:14,537 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:16,538 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:18,539 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:20,540 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:22,541 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:23,715 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:10:23,715 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:10:24,542 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:26,543 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:28,544 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:29,159 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:10:30,545 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:32,546 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:34,547 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:36,548 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:38,549 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:39,078 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:10:39,078 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:10:40,550 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:42,551 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:44,552 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:46,553 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:48,554 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:50,555 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:52,556 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:54,235 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:10:54,235 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:10:54,557 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:56,558 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:58,559 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:10:59,233 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:11:00,560 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:02,561 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:04,562 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:06,563 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:08,564 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:09,375 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:11:09,375 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:11:10,565 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:12,567 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:14,568 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:16,569 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:18,570 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:20,570 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:22,572 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:24,563 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:11:24,563 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:11:24,572 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:26,574 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:28,575 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:29,308 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:11:30,576 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:32,577 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:34,578 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:36,579 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:38,580 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:39,697 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:11:39,697 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:11:40,581 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:42,582 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:44,583 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:46,584 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:48,585 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:50,586 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:52,587 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:54,588 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:54,866 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:11:54,867 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:11:56,588 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:58,589 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:11:59,383 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:12:00,590 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:02,591 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:04,591 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:06,592 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:08,593 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:10,005 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:12:10,006 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:12:10,594 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:12,595 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:14,596 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:16,597 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:18,597 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:20,598 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:22,599 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:24,600 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:25,144 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:12:25,144 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:12:26,601 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:28,602 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:29,458 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:12:30,603 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:32,603 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:34,604 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:36,605 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:38,606 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:40,316 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status +2022-01-20 15:12:40,316 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status +2022-01-20 15:12:40,606 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:42,607 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:44,608 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:46,609 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:48,610 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:50,611 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:52,612 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:53,208 WARNING MainThread:2513451 [internal.py:wandb_internal():146] Internal process interrupt: 1 +2022-01-20 15:12:54,613 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:55,902 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:55,903 DEBUG SenderThread:2513451 [sender.py:send():234] send: telemetry +2022-01-20 15:12:55,903 DEBUG SenderThread:2513451 [sender.py:send():234] send: exit +2022-01-20 15:12:55,903 INFO SenderThread:2513451 [sender.py:send_exit():366] handling exit code: 255 +2022-01-20 15:12:55,903 INFO SenderThread:2513451 [sender.py:send_exit():368] handling runtime: 4565 +2022-01-20 15:12:55,904 INFO SenderThread:2513451 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-20 15:12:55,904 INFO SenderThread:2513451 [sender.py:send_exit():374] send defer +2022-01-20 15:12:55,904 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:55,905 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:55,905 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 0 +2022-01-20 15:12:55,905 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:55,905 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 0 +2022-01-20 15:12:55,905 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 1 +2022-01-20 15:12:55,905 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:55,905 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 1 +2022-01-20 15:12:55,924 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:55,924 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 1 +2022-01-20 15:12:55,924 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 2 +2022-01-20 15:12:55,924 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats +2022-01-20 15:12:55,925 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:55,925 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 2 +2022-01-20 15:12:55,925 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:55,925 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 2 +2022-01-20 15:12:55,925 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 3 +2022-01-20 15:12:55,926 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:55,926 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 3 +2022-01-20 15:12:55,926 DEBUG SenderThread:2513451 [sender.py:send():234] send: summary +2022-01-20 15:12:55,926 INFO SenderThread:2513451 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-20 15:12:55,926 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:55,926 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 3 +2022-01-20 15:12:55,926 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 4 +2022-01-20 15:12:55,927 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:55,927 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 4 +2022-01-20 15:12:55,927 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:55,927 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 4 +2022-01-20 15:12:56,006 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:56,102 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 5 +2022-01-20 15:12:56,102 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:56,103 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:56,103 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 5 +2022-01-20 15:12:56,103 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:56,103 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 5 +2022-01-20 15:12:56,103 INFO SenderThread:2513451 [dir_watcher.py:finish():283] shutting down directory watcher +2022-01-20 15:12:56,204 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:56,614 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml +2022-01-20 15:12:56,615 INFO SenderThread:2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:56,615 INFO SenderThread:2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json +2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():313] scan: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files +2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml config.yaml +2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/diff.patch diff.patch +2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/requirements.txt requirements.txt +2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log output.log +2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json wandb-summary.json +2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-metadata.json wandb-metadata.json +2022-01-20 15:12:56,617 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/code/run_mlm_flax.py code/run_mlm_flax.py +2022-01-20 15:12:56,617 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 6 +2022-01-20 15:12:56,618 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:56,622 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:56,622 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 6 +2022-01-20 15:12:56,622 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:56,622 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 6 +2022-01-20 15:12:56,622 INFO SenderThread:2513451 [file_pusher.py:finish():177] shutting down file pusher +2022-01-20 15:12:56,720 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:56,720 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:56,822 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:56,822 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:56,924 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:56,924 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:57,026 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:57,026 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:57,061 INFO Thread-15 :2513451 [upload_job.py:push():137] Uploaded file /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/requirements.txt +2022-01-20 15:12:57,074 INFO Thread-16 :2513451 [upload_job.py:push():137] Uploaded file /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +2022-01-20 15:12:57,078 INFO Thread-14 :2513451 [upload_job.py:push():137] Uploaded file /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml +2022-01-20 15:12:57,084 INFO Thread-17 :2513451 [upload_job.py:push():137] Uploaded file /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json +2022-01-20 15:12:57,128 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:57,128 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:57,230 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:57,230 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:57,285 INFO Thread-7 :2513451 [sender.py:transition_state():387] send defer: 7 +2022-01-20 15:12:57,286 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:57,286 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 7 +2022-01-20 15:12:57,286 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:57,286 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 7 +2022-01-20 15:12:57,332 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:57,712 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 8 +2022-01-20 15:12:57,713 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:57,713 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:57,713 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 8 +2022-01-20 15:12:57,714 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:57,714 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 8 +2022-01-20 15:12:57,714 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 9 +2022-01-20 15:12:57,714 DEBUG SenderThread:2513451 [sender.py:send():234] send: final +2022-01-20 15:12:57,714 DEBUG SenderThread:2513451 [sender.py:send():234] send: footer +2022-01-20 15:12:57,714 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer +2022-01-20 15:12:57,715 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 9 +2022-01-20 15:12:57,715 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer +2022-01-20 15:12:57,715 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 9 +2022-01-20 15:12:57,814 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit +2022-01-20 15:12:57,815 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit +2022-01-20 15:12:57,815 INFO SenderThread:2513451 [file_pusher.py:join():182] waiting for file pusher +2022-01-20 15:12:58,081 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: get_summary +2022-01-20 15:12:58,081 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: sampled_history +2022-01-20 15:12:58,082 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: shutdown +2022-01-20 15:12:58,082 INFO HandlerThread:2513451 [handler.py:finish():731] shutting down handler +2022-01-20 15:12:58,715 INFO WriterThread:2513451 [datastore.py:close():281] close: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/run-r5vgtpr6.wandb +2022-01-20 15:12:59,079 INFO SenderThread:2513451 [sender.py:finish():1070] shutting down sender +2022-01-20 15:12:59,080 INFO SenderThread:2513451 [file_pusher.py:finish():177] shutting down file pusher +2022-01-20 15:12:59,080 INFO SenderThread:2513451 [file_pusher.py:join():182] waiting for file pusher +2022-01-20 15:12:59,082 INFO MainThread:2513451 [internal.py:handle_exit():77] Internal process exited